2021-11-11T20:37:16.663 INFO:root:teuthology version: 1.1.0-d4737010 2021-11-11T20:37:16.665 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_soumyakoduri@teuthology --verbose --description rgw/multisite/{clusters frontend/beast ignore-pg-availability omap_limits overrides realms/three-zone-plus-pubsub supported-random-distro$/{rhel_8} tasks/test_multi} --name soumyakoduri-2021-11-11_20:19:05-rgw-cloudtiering-distro-basic-smithi --archive /home/teuthworker/archive/soumyakoduri-2021-11-11_20:19:05-rgw-cloudtiering-distro-basic-smithi/6497986 -- /home/teuthworker/archive/soumyakoduri-2021-11-11_20:19:05-rgw-cloudtiering-distro-basic-smithi/6497986/orig.config.yaml 2021-11-11T20:37:16.709 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2021-11-11T20:37:16.776 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/soumyakoduri-2021-11-11_20:19:05-rgw-cloudtiering-distro-basic-smithi/6497986 branch: cloudtiering description: rgw/multisite/{clusters frontend/beast ignore-pg-availability omap_limits overrides realms/three-zone-plus-pubsub supported-random-distro$/{rhel_8} tasks/test_multi} email: null first_in_suite: false job_id: '6497986' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: soumyakoduri-2021-11-11_20:19:05-rgw-cloudtiering-distro-basic-smithi nuke-on-error: true os_type: rhel os_version: '8.4' overrides: admin_socket: branch: cloudtiering ceph: conf: client: debug rgw: 20 rgw crypt require ssl: false rgw crypt s3 kms backend: testing rgw crypt s3 kms encryption keys: testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= rgw curl low speed time: 300 rgw data log num shards: 4 rgw md log max shards: 4 rgw sync log trim interval: 0 rgw sync meta inject err probability: 0.1 rgw sync obj etag verify: true setgroup: ceph setuser: ceph mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug filestore: 20 debug journal: 20 debug ms: 20 debug osd: 25 rocksdb delete range threshold: 0 osd.0: osd_max_omap_entries_per_request: 10 osd.1: osd_max_omap_entries_per_request: 1000 osd.2: osd_max_omap_entries_per_request: 10000 log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(PG_AVAILABILITY\) log-whitelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: a3a8018ba3cfe536252b397411bf18b7828c6098 wait-for-scrub: false ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 install: ceph: sha1: a3a8018ba3cfe536252b397411bf18b7828c6098 rgw: compression type: random frontend: beast rgw-multisite: realm: is default: true name: test-realm zonegroups: - endpoints: - c1.client.0 is_default: true is_master: true name: test-zonegroup zones: - endpoints: - c1.client.0 is_default: true is_master: true name: test-zone1 - endpoints: - c2.client.0 is_default: true name: test-zone2 - endpoints: - c1.client.1 name: test-zone3 - endpoints: - c2.client.1 is_pubsub: true name: test-zone4 rgw-multisite-tests: args: - tests.py - tests_ps.py selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: cloudtiering sha1: a3a8018ba3cfe536252b397411bf18b7828c6098 owner: scheduled_soumyakoduri@teuthology priority: 100 repo: https://github.com/ceph/ceph-ci.git roles: - - c1.mon.a - c1.mgr.x - c1.osd.0 - c1.osd.1 - c1.osd.2 - c1.client.0 - c1.client.1 - - c2.mon.a - c2.mgr.x - c2.osd.0 - c2.osd.1 - c2.osd.2 - c2.client.0 - c2.client.1 sha1: a3a8018ba3cfe536252b397411bf18b7828c6098 sleep_before_teardown: 0 suite: rgw suite_branch: cloudtiering suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_a3a8018ba3cfe536252b397411bf18b7828c6098/qa suite_relpath: qa suite_repo: git://git.ceph.com/ceph-ci.git suite_sha1: a3a8018ba3cfe536252b397411bf18b7828c6098 targets: smithi096.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCnnzgfZwb9DMmAqSFD0JEkotGtIng9gnT9CmKOgT3btn8RwZTgGqnrdK9xImGAZB25OR1Jma+qf1JWFmOpKBb+PipPVc6yGHbnCX0CLNS4vhkTTvQuBCxnu6Q6MOru6+5yzF3S3PK/bP24It0Crezuq6il9HCWMOf4m7AVYWp867OSYY+x29gpa58Ag2JcBysrC+LPPT0q0k60P6QGLgsp0mIxDEQUfx5e85+dWseIvmN5VzbOeSzuDMRuMr0GcqqTDJSnPI87JzequKN8CrYN6uXtyKZgvj/or7v1ycHdIWFn5MYz/eWRvv5Z5I0mlt8mSLr8U889gsVhH9kwe/e4I/DRdwjBY4WLRbtQe2YjTsTocusQdUwrHwSHuDyvQ/xj/GzvM7OxCI8E8Tq9n2bNWVB33r647D+QJxe42FioRqbdW8QBMAHA2ciBPK6bJ4iuL8AvtPJeD4GhoWgxpZRnrzgu0Xm7+MzcDJeoJ+3ygOrTHprj6AHz+0JMZoFtZEs= smithi198.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9jvdzDl2KeYL6DgYVb9e9leVYHjq3W9lqcBtDftr7EXr6wf0lNmnP+FikYaJOKq8PtBzr50l1o4lYLSSUhsZKS7GwWH1E1T95iY9FYTjv6M3LamGEygDOl905Y1OkkTmcrSWndwQIFh7yJnUVGUzhJxeDMICBoBclIsXjxAHYm4EYaePhsyZ3P6zAhIld/fpRSR3xgN5ggGmryOIHi+GHnN7qTYv+Qso1/vRKIcnFw7Q8GbRbx0wSb6sLwCEL4sA5O4CfLMmVsomQeD8wU8ze9voK/060tkeQtOo77ZTC8UEavnElU4Q/p85M7RQdiIf/lcJh7yN8NkieVh019SiiLPkgZ8D+rtrQsiV6vW5zv3VfC38cIi+l8rilwYomn/r4u6CJc0Og1bIp9yZLcS30RsoShNFxTLNjfBKYNz9IyFFfxtTZyDcPtGwxuZgQyfRhmhJJSwZlOMM7zjbHUB15z/l86+/pT9NYWX0WmHuwRXlQzUwAky053tlXYH0Ui4M= tasks: - install: null - ceph: cluster: c1 - ceph: cluster: c2 - rgw: c1.client.0: port: 8000 c1.client.1: port: 8001 c2.client.0: port: 8000 c2.client.1: port: 8001 - rgw-multisite: null - rgw-multisite-tests: config: reconfigure_delay: 60 teuthology_branch: master timestamp: 2021-11-11_20:19:05 tube: smithi user: soumyakoduri verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.20261 2021-11-11T20:37:16.777 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_a3a8018ba3cfe536252b397411bf18b7828c6098/qa; will attempt to use it 2021-11-11T20:37:16.778 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_a3a8018ba3cfe536252b397411bf18b7828c6098/qa/tasks 2021-11-11T20:37:16.778 INFO:teuthology.run_tasks:Running task internal.check_packages... 2021-11-11T20:37:16.781 INFO:teuthology.task.internal:Checking packages... 2021-11-11T20:37:16.804 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'a3a8018ba3cfe536252b397411bf18b7828c6098' 2021-11-11T20:37:16.805 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2021-11-11T20:37:16.805 INFO:teuthology.packaging:ref: None 2021-11-11T20:37:16.805 INFO:teuthology.packaging:tag: None 2021-11-11T20:37:16.806 INFO:teuthology.packaging:branch: cloudtiering 2021-11-11T20:37:16.806 INFO:teuthology.packaging:sha1: a3a8018ba3cfe536252b397411bf18b7828c6098 2021-11-11T20:37:16.807 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=cloudtiering 2021-11-11T20:37:17.034 INFO:teuthology.task.internal:Found packages for ceph version 17.0.0-8977.ga3a8018b 2021-11-11T20:37:17.034 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2021-11-11T20:37:17.045 INFO:teuthology.task.internal:no buildpackages task found 2021-11-11T20:37:17.046 INFO:teuthology.run_tasks:Running task internal.save_config... 2021-11-11T20:37:17.082 INFO:teuthology.task.internal:Saving configuration 2021-11-11T20:37:17.097 INFO:teuthology.run_tasks:Running task internal.check_lock... 2021-11-11T20:37:17.141 INFO:teuthology.task.internal.check_lock:Checking locks... 2021-11-11T20:37:17.163 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi096.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/soumyakoduri-2021-11-11_20:19:05-rgw-cloudtiering-distro-basic-smithi/6497986', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.4', 'arch': 'x86_64', 'locked': True, 'locked_since': '2021-11-11 20:32:02.318980', 'locked_by': 'scheduled_soumyakoduri@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGCwt654DVe+43ENMaaya1o+TjlLoeVlqjNseszIPKshPrNqJ5gdG7xzCl8NP65o1apJE9Pqh53dnuFPjHjs8Ss='} 2021-11-11T20:37:17.185 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi198.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/soumyakoduri-2021-11-11_20:19:05-rgw-cloudtiering-distro-basic-smithi/6497986', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.4', 'arch': 'x86_64', 'locked': True, 'locked_since': '2021-11-11 20:32:02.320322', 'locked_by': 'scheduled_soumyakoduri@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCLKxYAyg4ls1j+TkXbowFgG5+AfHvCoGZtQXxTSqPfpEC9qyphbXSHXXO8RbVz5c/CMSAdD1DkNJEUqYlzPxkU='} 2021-11-11T20:37:17.185 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2021-11-11T20:37:17.196 INFO:teuthology.task.internal:roles: ubuntu@smithi096.front.sepia.ceph.com - ['c1.mon.a', 'c1.mgr.x', 'c1.osd.0', 'c1.osd.1', 'c1.osd.2', 'c1.client.0', 'c1.client.1'] 2021-11-11T20:37:17.196 INFO:teuthology.task.internal:roles: ubuntu@smithi198.front.sepia.ceph.com - ['c2.mon.a', 'c2.mgr.x', 'c2.osd.0', 'c2.osd.1', 'c2.osd.2', 'c2.client.0', 'c2.client.1'] 2021-11-11T20:37:17.197 INFO:teuthology.run_tasks:Running task console_log... 2021-11-11T20:37:17.280 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7faeb0534b70>, signals=[15]) 2021-11-11T20:37:17.281 INFO:teuthology.run_tasks:Running task internal.connect... 2021-11-11T20:37:17.292 INFO:teuthology.task.internal:Opening connections... 2021-11-11T20:37:17.292 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi096.front.sepia.ceph.com 2021-11-11T20:37:17.294 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2021-11-11T20:37:17.362 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi198.front.sepia.ceph.com 2021-11-11T20:37:17.363 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi198.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2021-11-11T20:37:17.431 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2021-11-11T20:37:17.443 DEBUG:teuthology.orchestra.run.smithi096:> uname -m 2021-11-11T20:37:17.460 INFO:teuthology.orchestra.run.smithi096.stdout:x86_64 2021-11-11T20:37:17.461 DEBUG:teuthology.orchestra.run.smithi096:> cat /etc/os-release 2021-11-11T20:37:17.516 INFO:teuthology.orchestra.run.smithi096.stdout:NAME="Red Hat Enterprise Linux" 2021-11-11T20:37:17.517 INFO:teuthology.orchestra.run.smithi096.stdout:VERSION="8.4 (Ootpa)" 2021-11-11T20:37:17.517 INFO:teuthology.orchestra.run.smithi096.stdout:ID="rhel" 2021-11-11T20:37:17.517 INFO:teuthology.orchestra.run.smithi096.stdout:ID_LIKE="fedora" 2021-11-11T20:37:17.517 INFO:teuthology.orchestra.run.smithi096.stdout:VERSION_ID="8.4" 2021-11-11T20:37:17.517 INFO:teuthology.orchestra.run.smithi096.stdout:PLATFORM_ID="platform:el8" 2021-11-11T20:37:17.518 INFO:teuthology.orchestra.run.smithi096.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.4 (Ootpa)" 2021-11-11T20:37:17.518 INFO:teuthology.orchestra.run.smithi096.stdout:ANSI_COLOR="0;31" 2021-11-11T20:37:17.518 INFO:teuthology.orchestra.run.smithi096.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8.4:GA" 2021-11-11T20:37:17.518 INFO:teuthology.orchestra.run.smithi096.stdout:HOME_URL="https://www.redhat.com/" 2021-11-11T20:37:17.519 INFO:teuthology.orchestra.run.smithi096.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2021-11-11T20:37:17.519 INFO:teuthology.orchestra.run.smithi096.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2021-11-11T20:37:17.519 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:37:17.519 INFO:teuthology.orchestra.run.smithi096.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2021-11-11T20:37:17.520 INFO:teuthology.orchestra.run.smithi096.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.4 2021-11-11T20:37:17.520 INFO:teuthology.orchestra.run.smithi096.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2021-11-11T20:37:17.520 INFO:teuthology.orchestra.run.smithi096.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.4" 2021-11-11T20:37:17.521 INFO:teuthology.lock.ops:Updating smithi096.front.sepia.ceph.com on lock server 2021-11-11T20:37:17.539 DEBUG:teuthology.orchestra.run.smithi198:> uname -m 2021-11-11T20:37:17.555 INFO:teuthology.orchestra.run.smithi198.stdout:x86_64 2021-11-11T20:37:17.555 DEBUG:teuthology.orchestra.run.smithi198:> cat /etc/os-release 2021-11-11T20:37:17.610 INFO:teuthology.orchestra.run.smithi198.stdout:NAME="Red Hat Enterprise Linux" 2021-11-11T20:37:17.610 INFO:teuthology.orchestra.run.smithi198.stdout:VERSION="8.4 (Ootpa)" 2021-11-11T20:37:17.611 INFO:teuthology.orchestra.run.smithi198.stdout:ID="rhel" 2021-11-11T20:37:17.611 INFO:teuthology.orchestra.run.smithi198.stdout:ID_LIKE="fedora" 2021-11-11T20:37:17.611 INFO:teuthology.orchestra.run.smithi198.stdout:VERSION_ID="8.4" 2021-11-11T20:37:17.611 INFO:teuthology.orchestra.run.smithi198.stdout:PLATFORM_ID="platform:el8" 2021-11-11T20:37:17.612 INFO:teuthology.orchestra.run.smithi198.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.4 (Ootpa)" 2021-11-11T20:37:17.612 INFO:teuthology.orchestra.run.smithi198.stdout:ANSI_COLOR="0;31" 2021-11-11T20:37:17.612 INFO:teuthology.orchestra.run.smithi198.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8.4:GA" 2021-11-11T20:37:17.612 INFO:teuthology.orchestra.run.smithi198.stdout:HOME_URL="https://www.redhat.com/" 2021-11-11T20:37:17.613 INFO:teuthology.orchestra.run.smithi198.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2021-11-11T20:37:17.613 INFO:teuthology.orchestra.run.smithi198.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2021-11-11T20:37:17.613 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:37:17.613 INFO:teuthology.orchestra.run.smithi198.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2021-11-11T20:37:17.614 INFO:teuthology.orchestra.run.smithi198.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.4 2021-11-11T20:37:17.614 INFO:teuthology.orchestra.run.smithi198.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2021-11-11T20:37:17.614 INFO:teuthology.orchestra.run.smithi198.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.4" 2021-11-11T20:37:17.614 INFO:teuthology.lock.ops:Updating smithi198.front.sepia.ceph.com on lock server 2021-11-11T20:37:17.633 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2021-11-11T20:37:17.647 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2021-11-11T20:37:17.666 INFO:teuthology.task.internal:Checking for old test directory... 2021-11-11T20:37:17.667 DEBUG:teuthology.orchestra.run.smithi096:> test '!' -e /home/ubuntu/cephtest 2021-11-11T20:37:17.670 DEBUG:teuthology.orchestra.run.smithi198:> test '!' -e /home/ubuntu/cephtest 2021-11-11T20:37:17.685 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2021-11-11T20:37:17.692 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2021-11-11T20:37:17.693 DEBUG:teuthology.orchestra.run.smithi096:> test -z $(ls -A /var/lib/ceph) 2021-11-11T20:37:17.727 DEBUG:teuthology.orchestra.run.smithi198:> test -z $(ls -A /var/lib/ceph) 2021-11-11T20:37:17.747 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2021-11-11T20:37:17.804 INFO:teuthology.run_tasks:Running task kernel... 2021-11-11T20:37:17.820 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2021-11-11T20:37:17.821 INFO:teuthology.task.kernel:config {'c1.mon.a': {'kdb': True, 'sha1': 'distro'}, 'c2.mon.a': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2021-11-11T20:37:17.822 DEBUG:teuthology.orchestra.run.smithi096:> uname -r 2021-11-11T20:37:17.838 INFO:teuthology.orchestra.run.smithi096.stdout:4.18.0-305.el8.x86_64 2021-11-11T20:37:17.839 INFO:teuthology.task.kernel:Running kernel on smithi096: 4.18.0-305.el8.x86_64 2021-11-11T20:37:17.839 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum install -y kernel 2021-11-11T20:37:19.190 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:37:19.191 INFO:teuthology.orchestra.run.smithi096.stdout:Unable to read consumer identity 2021-11-11T20:37:19.327 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:37:19.328 INFO:teuthology.orchestra.run.smithi096.stdout:This system is not registered to Red Hat Subscription Management. You can use subscription-manager to register. 2021-11-11T20:37:19.328 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:37:21.158 INFO:teuthology.orchestra.run.smithi096.stdout:Extra Packages for Enterprise Linux 16 MB/s | 11 MB 00:00 2021-11-11T20:37:23.328 INFO:teuthology.orchestra.run.smithi096.stdout:lab-extras 4.6 MB/s | 24 kB 00:00 2021-11-11T20:37:23.911 INFO:teuthology.orchestra.run.smithi096.stdout:Package kernel-4.18.0-305.el8.x86_64 is already installed. 2021-11-11T20:37:24.023 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:37:24.024 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2021-11-11T20:37:24.025 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:37:24.140 DEBUG:teuthology.orchestra.run.smithi096:> echo no | sudo yum reinstall kernel || true 2021-11-11T20:37:24.496 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:37:24.497 INFO:teuthology.orchestra.run.smithi096.stdout:Unable to read consumer identity 2021-11-11T20:37:24.561 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:37:24.561 INFO:teuthology.orchestra.run.smithi096.stdout:This system is not registered to Red Hat Subscription Management. You can use subscription-manager to register. 2021-11-11T20:37:24.562 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:37:24.741 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:01 ago on Thu 11 Nov 2021 08:37:23 PM UTC. 2021-11-11T20:37:24.851 INFO:teuthology.orchestra.run.smithi096.stdout:Installed package kernel-4.18.0-305.el8.x86_64 (from anaconda) not available. 2021-11-11T20:37:24.881 INFO:teuthology.orchestra.run.smithi096.stderr:Error: No packages marked for reinstall. 2021-11-11T20:37:24.924 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum reinstall -y kernel || true 2021-11-11T20:37:25.236 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:37:25.237 INFO:teuthology.orchestra.run.smithi096.stdout:Unable to read consumer identity 2021-11-11T20:37:25.288 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:37:25.288 INFO:teuthology.orchestra.run.smithi096.stdout:This system is not registered to Red Hat Subscription Management. You can use subscription-manager to register. 2021-11-11T20:37:25.288 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:37:25.462 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:02 ago on Thu 11 Nov 2021 08:37:23 PM UTC. 2021-11-11T20:37:25.582 INFO:teuthology.orchestra.run.smithi096.stdout:Installed package kernel-4.18.0-305.el8.x86_64 (from anaconda) not available. 2021-11-11T20:37:25.605 INFO:teuthology.orchestra.run.smithi096.stderr:Error: No packages marked for reinstall. 2021-11-11T20:37:25.649 DEBUG:teuthology.orchestra.run.smithi096:> rpm -q kernel --last | head -n 1 2021-11-11T20:37:25.713 INFO:teuthology.orchestra.run.smithi096.stdout:kernel-4.18.0-305.el8.x86_64 Thu 20 May 2021 08:07:00 PM UTC 2021-11-11T20:37:25.715 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-305.el8.x86_64 2021-11-11T20:37:25.715 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2021-11-11T20:37:25.716 DEBUG:teuthology.orchestra.run.smithi198:> uname -r 2021-11-11T20:37:25.733 INFO:teuthology.orchestra.run.smithi198.stdout:4.18.0-305.el8.x86_64 2021-11-11T20:37:25.733 INFO:teuthology.task.kernel:Running kernel on smithi198: 4.18.0-305.el8.x86_64 2021-11-11T20:37:25.733 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum install -y kernel 2021-11-11T20:37:27.159 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:37:27.159 INFO:teuthology.orchestra.run.smithi198.stdout:Unable to read consumer identity 2021-11-11T20:37:27.248 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:37:27.248 INFO:teuthology.orchestra.run.smithi198.stdout:This system is not registered to Red Hat Subscription Management. You can use subscription-manager to register. 2021-11-11T20:37:27.248 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:37:29.216 INFO:teuthology.orchestra.run.smithi198.stdout:Extra Packages for Enterprise Linux 15 MB/s | 11 MB 00:00 2021-11-11T20:37:31.320 INFO:teuthology.orchestra.run.smithi198.stdout:lab-extras 1.2 MB/s | 24 kB 00:00 2021-11-11T20:37:31.875 INFO:teuthology.orchestra.run.smithi198.stdout:Package kernel-4.18.0-305.el8.x86_64 is already installed. 2021-11-11T20:37:31.989 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:37:31.991 INFO:teuthology.orchestra.run.smithi198.stdout:Nothing to do. 2021-11-11T20:37:31.991 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:37:32.116 DEBUG:teuthology.orchestra.run.smithi198:> echo no | sudo yum reinstall kernel || true 2021-11-11T20:37:32.469 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:37:32.469 INFO:teuthology.orchestra.run.smithi198.stdout:Unable to read consumer identity 2021-11-11T20:37:32.530 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:37:32.531 INFO:teuthology.orchestra.run.smithi198.stdout:This system is not registered to Red Hat Subscription Management. You can use subscription-manager to register. 2021-11-11T20:37:32.531 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:37:32.712 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:01 ago on Thu 11 Nov 2021 08:37:31 PM UTC. 2021-11-11T20:37:32.823 INFO:teuthology.orchestra.run.smithi198.stdout:Installed package kernel-4.18.0-305.el8.x86_64 (from anaconda) not available. 2021-11-11T20:37:32.858 INFO:teuthology.orchestra.run.smithi198.stderr:Error: No packages marked for reinstall. 2021-11-11T20:37:32.902 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum reinstall -y kernel || true 2021-11-11T20:37:33.215 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:37:33.215 INFO:teuthology.orchestra.run.smithi198.stdout:Unable to read consumer identity 2021-11-11T20:37:33.261 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:37:33.262 INFO:teuthology.orchestra.run.smithi198.stdout:This system is not registered to Red Hat Subscription Management. You can use subscription-manager to register. 2021-11-11T20:37:33.262 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:37:33.438 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:02 ago on Thu 11 Nov 2021 08:37:31 PM UTC. 2021-11-11T20:37:33.576 INFO:teuthology.orchestra.run.smithi198.stdout:Installed package kernel-4.18.0-305.el8.x86_64 (from anaconda) not available. 2021-11-11T20:37:33.596 INFO:teuthology.orchestra.run.smithi198.stderr:Error: No packages marked for reinstall. 2021-11-11T20:37:33.640 DEBUG:teuthology.orchestra.run.smithi198:> rpm -q kernel --last | head -n 1 2021-11-11T20:37:33.713 INFO:teuthology.orchestra.run.smithi198.stdout:kernel-4.18.0-305.el8.x86_64 Thu 20 May 2021 08:07:00 PM UTC 2021-11-11T20:37:33.714 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-305.el8.x86_64 2021-11-11T20:37:33.715 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2021-11-11T20:37:33.715 INFO:teuthology.task.kernel:Enabling kdb on c1.mon.a... 2021-11-11T20:37:33.715 DEBUG:teuthology.orchestra.run.smithi096:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2021-11-11T20:37:33.748 INFO:teuthology.orchestra.run.smithi096.stdout:ttyS1 2021-11-11T20:37:33.762 INFO:teuthology.task.kernel:Enabling kdb on c2.mon.a... 2021-11-11T20:37:33.762 DEBUG:teuthology.orchestra.run.smithi198:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2021-11-11T20:37:33.817 INFO:teuthology.orchestra.run.smithi198.stdout:ttyS1 2021-11-11T20:37:33.829 INFO:teuthology.run_tasks:Running task internal.base... 2021-11-11T20:37:33.841 INFO:teuthology.task.internal:Creating test directory... 2021-11-11T20:37:33.842 DEBUG:teuthology.orchestra.run.smithi096:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2021-11-11T20:37:33.845 DEBUG:teuthology.orchestra.run.smithi198:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2021-11-11T20:37:33.861 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2021-11-11T20:37:33.881 INFO:teuthology.run_tasks:Running task internal.archive... 2021-11-11T20:37:33.905 INFO:teuthology.task.internal:Creating archive directory... 2021-11-11T20:37:33.906 DEBUG:teuthology.orchestra.run.smithi096:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2021-11-11T20:37:33.909 DEBUG:teuthology.orchestra.run.smithi198:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2021-11-11T20:37:33.946 INFO:teuthology.run_tasks:Running task internal.coredump... 2021-11-11T20:37:33.957 INFO:teuthology.task.internal:Enabling coredump saving... 2021-11-11T20:37:33.958 DEBUG:teuthology.orchestra.run.smithi096:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2021-11-11T20:37:33.985 DEBUG:teuthology.orchestra.run.smithi198:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2021-11-11T20:37:34.019 INFO:teuthology.orchestra.run.smithi096.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2021-11-11T20:37:34.020 INFO:teuthology.orchestra.run.smithi198.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2021-11-11T20:37:34.030 INFO:teuthology.orchestra.run.smithi198.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2021-11-11T20:37:34.031 INFO:teuthology.orchestra.run.smithi096.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2021-11-11T20:37:34.032 INFO:teuthology.run_tasks:Running task internal.sudo... 2021-11-11T20:37:34.045 INFO:teuthology.task.internal:Configuring sudo... 2021-11-11T20:37:34.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2021-11-11T20:37:34.075 DEBUG:teuthology.orchestra.run.smithi198:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2021-11-11T20:37:34.100 INFO:teuthology.run_tasks:Running task internal.syslog... 2021-11-11T20:37:34.112 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2021-11-11T20:37:34.113 DEBUG:teuthology.orchestra.run.smithi096:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2021-11-11T20:37:34.143 DEBUG:teuthology.orchestra.run.smithi198:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2021-11-11T20:37:34.157 DEBUG:teuthology.orchestra.run.smithi096:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2021-11-11T20:37:34.236 DEBUG:teuthology.orchestra.run.smithi096:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2021-11-11T20:37:34.287 DEBUG:teuthology.orchestra.run.smithi096:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2021-11-11T20:37:34.367 DEBUG:teuthology.orchestra.run.smithi096:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2021-11-11T20:37:34.409 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:37:34.410 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2021-11-11T20:37:34.473 DEBUG:teuthology.orchestra.run.smithi198:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2021-11-11T20:37:34.511 DEBUG:teuthology.orchestra.run.smithi198:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2021-11-11T20:37:34.578 DEBUG:teuthology.orchestra.run.smithi198:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2021-11-11T20:37:34.616 DEBUG:teuthology.orchestra.run.smithi198:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2021-11-11T20:37:34.657 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:37:34.658 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2021-11-11T20:37:34.722 DEBUG:teuthology.orchestra.run.smithi096:> sudo service rsyslog restart 2021-11-11T20:37:34.725 DEBUG:teuthology.orchestra.run.smithi198:> sudo service rsyslog restart 2021-11-11T20:37:34.772 INFO:teuthology.orchestra.run.smithi096.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2021-11-11T20:37:34.807 INFO:teuthology.orchestra.run.smithi198.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2021-11-11T20:37:35.067 INFO:teuthology.run_tasks:Running task internal.timer... 2021-11-11T20:37:35.079 INFO:teuthology.task.internal:Starting timer... 2021-11-11T20:37:35.080 INFO:teuthology.run_tasks:Running task pcp... 2021-11-11T20:37:35.113 INFO:teuthology.run_tasks:Running task selinux... 2021-11-11T20:37:35.133 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2021-11-11T20:37:35.134 DEBUG:teuthology.orchestra.run.smithi096:> sudo service auditd rotate 2021-11-11T20:37:35.194 INFO:teuthology.orchestra.run.smithi096.stdout:Rotating logs: [ OK ] 2021-11-11T20:37:35.195 DEBUG:teuthology.orchestra.run.smithi198:> sudo service auditd rotate 2021-11-11T20:37:35.254 INFO:teuthology.orchestra.run.smithi198.stdout:Rotating logs: [ OK ] 2021-11-11T20:37:35.256 DEBUG:teuthology.task.selinux:Getting current SELinux state 2021-11-11T20:37:35.256 DEBUG:teuthology.orchestra.run.smithi096:> /usr/sbin/getenforce 2021-11-11T20:37:35.280 INFO:teuthology.orchestra.run.smithi096.stdout:Permissive 2021-11-11T20:37:35.281 DEBUG:teuthology.orchestra.run.smithi198:> /usr/sbin/getenforce 2021-11-11T20:37:35.304 INFO:teuthology.orchestra.run.smithi198.stdout:Permissive 2021-11-11T20:37:35.305 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi096.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi198.front.sepia.ceph.com': 'permissive'} 2021-11-11T20:37:35.305 DEBUG:teuthology.orchestra.run.smithi096:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|comm="sssd"\|scontext=system_u:system_r:logrotate_t:s0\)' 2021-11-11T20:37:35.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-11T20:37:35.345 DEBUG:teuthology.orchestra.run.smithi198:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|comm="sssd"\|scontext=system_u:system_r:logrotate_t:s0\)' 2021-11-11T20:37:35.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-11T20:37:35.370 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2021-11-11T20:37:35.370 DEBUG:teuthology.orchestra.run.smithi096:> sudo /usr/sbin/setenforce permissive 2021-11-11T20:37:35.413 DEBUG:teuthology.orchestra.run.smithi198:> sudo /usr/sbin/setenforce permissive 2021-11-11T20:37:35.438 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2021-11-11T20:37:35.449 DEBUG:teuthology.repo_utils:Setting repo remote to git://git.ceph.com/git/ceph-cm-ansible.git 2021-11-11T20:37:35.456 INFO:teuthology.repo_utils:Fetching master from origin 2021-11-11T20:37:35.492 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master to origin/master 2021-11-11T20:37:35.511 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2021-11-11T20:37:35.512 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi096.front.sepia.ceph.com,smithi198.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/cephlab.yml 2021-11-11T20:37:35.928 INFO:teuthology.task.ansible.out:Using /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/ansible.cfg as config file 2021-11-11T20:37:36.773 INFO:teuthology.task.ansible.out:[WARNING]: file /home/teuthworker/src/git.ceph.com_git_ceph-cm- ansible_master/roles/testnode/tasks/setup-opensuse.yml is empty and had no tasks to include 2021-11-11T20:37:39.772 INFO:teuthology.task.ansible.out: PLAY [all] ********************************************************************* 2021-11-11T20:37:39.904 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:39 +0000 (0:00:00.160) 0:00:00.160 ***** 2021-11-11T20:37:39.916 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:39 +0000 (0:00:00.012) 0:00:00.172 ***** 2021-11-11T20:37:42.269 INFO:teuthology.task.ansible.out: TASK [Gathering Facts] ********************************************************* 2021-11-11T20:37:42.270 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] 2021-11-11T20:37:42.278 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] 2021-11-11T20:37:42.335 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:42 +0000 (0:00:02.418) 0:00:02.591 ***** 2021-11-11T20:37:42.355 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:42 +0000 (0:00:00.019) 0:00:02.610 ***** 2021-11-11T20:37:42.609 INFO:teuthology.task.ansible.out: TASK [ansible-managed : Create the sudo group.] ******************************** 2021-11-11T20:37:42.611 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "gid": 1000, "name": "sudo", "state": "present", "system": false} ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "gid": 1000, "name": "sudo", "state": "present", "system": false} 2021-11-11T20:37:42.630 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:42 +0000 (0:00:00.275) 0:00:02.886 ***** 2021-11-11T20:37:42.646 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:42 +0000 (0:00:00.016) 0:00:02.902 ***** 2021-11-11T20:37:43.135 INFO:teuthology.task.ansible.out: TASK [ansible-managed : Create the ansible user.] ****************************** 2021-11-11T20:37:43.137 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"append": false, "changed": false, "comment": "", "group": 1100, "groups": "sudo", "home": "/home/cm", "move_home": false, "name": "cm", "shell": "/bin/bash", "state": "present", "uid": 1100} 2021-11-11T20:37:43.158 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:43 +0000 (0:00:00.512) 0:00:03.414 ***** 2021-11-11T20:37:43.166 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"append": false, "changed": false, "comment": "", "group": 1100, "groups": "sudo", "home": "/home/cm", "move_home": false, "name": "cm", "shell": "/bin/bash", "state": "present", "uid": 1100} 2021-11-11T20:37:43.190 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:43 +0000 (0:00:00.032) 0:00:03.446 ***** 2021-11-11T20:37:43.229 INFO:teuthology.task.ansible.out: TASK [ansible-managed : Delete the ansible users password.] ******************** 2021-11-11T20:37:43.230 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:43.250 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:43 +0000 (0:00:00.059) 0:00:03.506 ***** 2021-11-11T20:37:43.257 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:43.279 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:43 +0000 (0:00:00.028) 0:00:03.535 ***** 2021-11-11T20:37:43.503 INFO:teuthology.task.ansible.out: TASK [ansible-managed : Ensure includedir is present in sudoers.] ************** 2021-11-11T20:37:43.505 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"backup": "", "changed": false, "msg": ""} ok: [smithi198.front.sepia.ceph.com] => {"backup": "", "changed": false, "msg": ""} 2021-11-11T20:37:43.529 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:43 +0000 (0:00:00.250) 0:00:03.785 ***** 2021-11-11T20:37:43.548 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:43 +0000 (0:00:00.018) 0:00:03.804 ***** 2021-11-11T20:37:44.222 INFO:teuthology.task.ansible.out: TASK [ansible-managed : Create the cephlab_sudo sudoers.d file.] *************** 2021-11-11T20:37:44.224 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "checksum": "6c8e8b7ecc895eb2094cafe75c34b58f183442a7", "dest": "/etc/sudoers.d/cephlab_sudo", "gid": 0, "group": "root", "md5sum": "c3657b9ef874d0ad278742c3210136a0", "mode": "0440", "owner": "root", "secontext": "system_u:object_r:etc_t:s0", "size": 182, "src": "/home/ubuntu/.ansible/tmp/ansible-tmp-1636663063.5909991-5521-167483885035862/source", "state": "file", "uid": 0} changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "checksum": "6c8e8b7ecc895eb2094cafe75c34b58f183442a7", "dest": "/etc/sudoers.d/cephlab_sudo", "gid": 0, "group": "root", "md5sum": "c3657b9ef874d0ad278742c3210136a0", "mode": "0440", "owner": "root", "secontext": "system_u:object_r:etc_t:s0", "size": 182, "src": "/home/ubuntu/.ansible/tmp/ansible-tmp-1636663063.5743432-5519-243378779969792/source", "state": "file", "uid": 0} 2021-11-11T20:37:44.244 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:44 +0000 (0:00:00.696) 0:00:04.500 ***** 2021-11-11T20:37:44.262 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:44 +0000 (0:00:00.018) 0:00:04.518 ***** 2021-11-11T20:37:44.618 INFO:teuthology.task.ansible.out: TASK [ansible-managed : Add authorized keys for the ansible user.] ************* 2021-11-11T20:37:44.620 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "comment": null, "exclusive": true, "follow": false, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDmiHwkLkoJZiQs67KfE8jf0CdXjgbbpzKywUleMk4kWC7Wdn+92q/33DXpfS8xjg7opB9GTK3EW2C/RKBpf5RdaVG/5M7tRyUDLLbS3CURUoF6aNQI+8JIuFlsIfhO3Qw4uT7NlJCqsusrSbyLtGd/Im4CzwhvP1Enn6aUfyxv2yV4HoVjXbbIp2hlRXgcW8G/9q0TMuNfX4J6dpj9/PG8BbkKH/NReL2UMtjCGyRdIQvIwJRHMozjcE21824ZatA7nBeecX0MBRS6/A286uj6Z0PgmfJXYw0yw2Z2uSWR1ptIuMEiDBt96Pzn0dvfl7g/7bCi93MBQf/6sH9n4wkN root@typica001\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDehvF7TbkfX9Ga4fNdq/j9GHu3c9cNk4iIHVs7e14j0h9DlV14aPyn/M0+z90PK+5p2uL9EvRzkbsPrfxpG3pxUa1pmVmKIPvBruA7ln8tExOVwO3XEsBG7b8VWUMlQmwRS9WEfkKAIHlr9cgIFGb1yrMW30RyL7keO1giotAq/wR9pMicyqYAbZOtI3MuPsDVTNxtyD3RwRUQicz6vNaK/RwfMCZVgdvoinrJ7VSyFVEiitzuQqMlrF935sfLvhuyxmwvJKfZI8QMEYcFbKd1cDnCk9n/PySNRY96gb41nb5pUk+0eaDu37jnmj7LLqoidD3UWUuHNnGBAAk+zNlD andrew.schoen@gmail.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBwJCoMbbxQrI9jM5I/lX0MYDB57LzYXDJTXqmLfJ8E5IHwQbn708/EFXAHuJlCyoJAuA8FaU/Y5/l2LBlVXwp9v+8ftk/D6AcsuQ+Hi0ZLCTvW10OUIS2cwX/of//JJIu6roy3r3SWwUcQvwjaZYLm7QkC+1StsLNMHdgiVRFiibqml72Q53VTtSTt6z2aXjrHzhpZq0hBK/13aknzAjHvGprQnchKVzhA/7A5pArF3CXpnI7aCwKvvjU0AcvqqD8WNLMpF4hFSoTK7xluKB+5UAJ36wefIO2KFM/zMwUtl4/aDhDdPeSaQRWq/q7viKLVNS0X9j2PjSRTfYhJNZr zack@zack-cerzas-macbook-pro.local\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC305rmql/Qm8Si+DFvAs77hSUZakwiXhhAhSetS89bWq3AdPQTL1MUXJHFm3AMzwzb00tndvsl1AUmFseXLfAi87daNcKJdwEj/JvXgcHKzAXOpVRfArHh08JnOumQiP4EoLYvq7rL+3LjEFURKOWJLLnGxh4E+iiJAnxLTG3WuUcnz9kXVTwtmIs60+2yH+gZJkkB04mnivPYjX46sHigoKgtlE2JnQfOU1lBkeZaBeUhoZX2rIKwRyGELPPcUBwHk10izyk6tePLAw9bSWeS9uEaa/m+qa8neJDyR4/hWQNR9yRXp0YjKkiKoFog+qtsBt3xBNzTPsFLxZ5zugmp dmick@angus\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKT1KilDopHJzDZ90QJx4+7WVIC4WNdvpD6VrS83B3PFOplNyXZLfDSDRZ4wEzIKLwtR9vLWWInVBNFU00W/2dObSC+l9GDzKsHtO3KSVmAjHEk6ZAJ5Mc3eUPR0lAywpFD8+yg+kDDIkRtCtoHDq+kFvNQutdvrMuhNWOZLyQ2N2XrpRJVD2kiJvaml6nQoZXMtVQGPVcYbn3TRed/wxT/han2u6j/wiElv 2021-11-11T20:37:44.622 INFO:teuthology.task.ansible.out:4r9ty+9zi3P1MoNiALD53sD7wMr/5Si6PX+5zWcXTNcQmTuGpiLwGw0RxCwr4gtwqKBaNae2d1bwnHanLc1KvQEw29O6aBca2f7GWxFY6QtQV/mNft dgalloway@w541\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDoEI+g74GcJe/03nDypMzDWqg93K1h13xQPFvCIN7ElNZ+ulY5av5KLnv/gmLbYGRPlgxC7LNZXE7YQPUo2DfA8awnDkAkE/nWvL4fQjUxmaiPthjIWsKvk9EvPIqzLjDbwsiLiRPE0Sxka0qBa1GHU1QYN6cw7l9nWF0dG/TsKw+QmuArBKMgm9Th/QVLq1ARHC/07M1n195QvBjdrKSzOUl+2L1tADHg++sEWDq/MJMiylyNdxp79Wz7aTIlR6l8NV+Ega9LUb6B6xmO7k+J+/hJF7gYOeFl+YqnZttEdz+o8FbLjhTjOF9QkBsGWlrawXtQln/cVCryV+LAt57L dgalloway@p50\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDo3gswIiZ2xPWMUQNjV1Gd4gYSxVpJM8Jb5pH1lqvoKAsmhVI30qTYxIaezzDWfSYwdkNYQAC9eM5v7EMQrOI7eZtTzCH+gbF79GciGCLL2bDze1I8hJ+jUu4bv+wnmvKsHyyOVBBJPSJ+eGFCzsu7vOiK4iFX0zgCXYwvP+A4Y/+kVR1en39gMtD6Hrw8ACBhxt+C2G2zc1807NRj3pIw1GvQILa9Sm3VKdNSrHzGB+MBooUoRmmjjzZ/bRwHPWVj+sX+xkoX4P2zwUcfMgZ/f4GgSgHyEvS3oxqbSaZp0pidZyoCaLRgqVPiUgXQEm0HA5PDDULAthQOfxQveKB/ akraitma@aklap", "key_options": null, "keyfile": "/home/cm/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "cm", "validate_certs": true} 2021-11-11T20:37:44.669 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "comment": null, "exclusive": true, "follow": false, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDmiHwkLkoJZiQs67KfE8jf0CdXjgbbpzKywUleMk4kWC7Wdn+92q/33DXpfS8xjg7opB9GTK3EW2C/RKBpf5RdaVG/5M7tRyUDLLbS3CURUoF6aNQI+8JIuFlsIfhO3Qw4uT7NlJCqsusrSbyLtGd/Im4CzwhvP1Enn6aUfyxv2yV4HoVjXbbIp2hlRXgcW8G/9q0TMuNfX4J6dpj9/PG8BbkKH/NReL2UMtjCGyRdIQvIwJRHMozjcE21824ZatA7nBeecX0MBRS6/A286uj6Z0PgmfJXYw0yw2Z2uSWR1ptIuMEiDBt96Pzn0dvfl7g/7bCi93MBQf/6sH9n4wkN root@typica001\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDehvF7TbkfX9Ga4fNdq/j9GHu3c9cNk4iIHVs7e14j0h9DlV14aPyn/M0+z90PK+5p2uL9EvRzkbsPrfxpG3pxUa1pmVmKIPvBruA7ln8tExOVwO3XEsBG7b8VWUMlQmwRS9WEfkKAIHlr9cgIFGb1yrMW30RyL7keO1giotAq/wR9pMicyqYAbZOtI3MuPsDVTNxtyD3RwRUQicz6vNaK/RwfMCZVgdvoinrJ7VSyFVEiitzuQqMlrF935sfLvhuyxmwvJKfZI8QMEYcFbKd1cDnCk9n/PySNRY96gb41nb5pUk+0eaDu37jnmj7LLqoidD3UWUuHNnGBAAk+zNlD andrew.schoen@gmail.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBwJCoMbbxQrI9jM5I/lX0MYDB57LzYXDJTXqmLfJ8E5IHwQbn708/EFXAHuJlCyoJAuA8FaU/Y5/l2LBlVXwp9v+8ftk/D6AcsuQ+Hi0ZLCTvW10OUIS2cwX/of//JJIu6roy3r3SWwUcQvwjaZYLm7QkC+1StsLNMHdgiVRFiibqml72Q53VTtSTt6z2aXjrHzhpZq0hBK/13aknzAjHvGprQnchKVzhA/7A5pArF3CXpnI7aCwKvvjU0AcvqqD8WNLMpF4hFSoTK7xluKB+5UAJ36wefIO2KFM/zMwUtl4/aDhDdPeSaQRWq/q7viKLVNS0X9j2PjSRTfYhJNZr zack@zack-cerzas-macbook-pro.local\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC305rmql/Qm8Si+DFvAs77hSUZakwiXhhAhSetS89bWq3AdPQTL1MUXJHFm3AMzwzb00tndvsl1AUmFseXLfAi87daNcKJdwEj/JvXgcHKzAXOpVRfArHh08JnOumQiP4EoLYvq7rL+3LjEFURKOWJLLnGxh4E+iiJAnxLTG3WuUcnz9kXVTwtmIs60+2yH+gZJkkB04mnivPYjX46sHigoKgtlE2JnQfOU1lBkeZaBeUhoZX2rIKwRyGELPPcUBwHk10izyk6tePLAw9bSWeS9uEaa/m+qa8neJDyR4/hWQNR9yRXp0YjKkiKoFog+qtsBt3xBNzTPsFLxZ5zugmp dmick@angus\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKT1KilDopHJzDZ90QJx4+7WVIC4WNdvpD6VrS83B3PFOplNyXZLfDSDRZ4wEzIKLwtR9vLWWInVBNFU00W/2dObSC+l9GDzKsHtO3KSVmAjHEk6ZAJ5Mc3eUPR0lAywpFD8+yg+kDDIkRtCtoHDq+kFvNQutdvrMuhNWOZLyQ2N2XrpRJVD2kiJvaml6nQoZXMtVQGPVcYbn3TRed/wxT/han2u6j/wiElv 2021-11-11T20:37:44.671 INFO:teuthology.task.ansible.out:4r9ty+9zi3P1MoNiALD53sD7wMr/5Si6PX+5zWcXTNcQmTuGpiLwGw0RxCwr4gtwqKBaNae2d1bwnHanLc1KvQEw29O6aBca2f7GWxFY6QtQV/mNft dgalloway@w541\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDoEI+g74GcJe/03nDypMzDWqg93K1h13xQPFvCIN7ElNZ+ulY5av5KLnv/gmLbYGRPlgxC7LNZXE7YQPUo2DfA8awnDkAkE/nWvL4fQjUxmaiPthjIWsKvk9EvPIqzLjDbwsiLiRPE0Sxka0qBa1GHU1QYN6cw7l9nWF0dG/TsKw+QmuArBKMgm9Th/QVLq1ARHC/07M1n195QvBjdrKSzOUl+2L1tADHg++sEWDq/MJMiylyNdxp79Wz7aTIlR6l8NV+Ega9LUb6B6xmO7k+J+/hJF7gYOeFl+YqnZttEdz+o8FbLjhTjOF9QkBsGWlrawXtQln/cVCryV+LAt57L dgalloway@p50\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDo3gswIiZ2xPWMUQNjV1Gd4gYSxVpJM8Jb5pH1lqvoKAsmhVI30qTYxIaezzDWfSYwdkNYQAC9eM5v7EMQrOI7eZtTzCH+gbF79GciGCLL2bDze1I8hJ+jUu4bv+wnmvKsHyyOVBBJPSJ+eGFCzsu7vOiK4iFX0zgCXYwvP+A4Y/+kVR1en39gMtD6Hrw8ACBhxt+C2G2zc1807NRj3pIw1GvQILa9Sm3VKdNSrHzGB+MBooUoRmmjjzZ/bRwHPWVj+sX+xkoX4P2zwUcfMgZ/f4GgSgHyEvS3oxqbSaZp0pidZyoCaLRgqVPiUgXQEm0HA5PDDULAthQOfxQveKB/ akraitma@aklap", "key_options": null, "keyfile": "/home/cm/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "cm", "validate_certs": true} 2021-11-11T20:37:44.731 INFO:teuthology.task.ansible.out: PLAY [teuthology] ************************************************************** skipping: no hosts matched 2021-11-11T20:37:44.733 INFO:teuthology.task.ansible.out: PLAY [testnodes] *************************************************************** 2021-11-11T20:37:44.748 INFO:teuthology.task.ansible.out: TASK [Gathering Facts] ********************************************************* 2021-11-11T20:37:44.750 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:44 +0000 (0:00:00.485) 0:00:05.004 ***** 2021-11-11T20:37:45.266 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] 2021-11-11T20:37:45.284 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] 2021-11-11T20:37:45.300 INFO:teuthology.task.ansible.out: TASK [set_fact] **************************************************************** 2021-11-11T20:37:45.301 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:45 +0000 (0:00:00.551) 0:00:05.556 ***** 2021-11-11T20:37:45.322 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"ran_from_cephlab_playbook": true}, "changed": false} 2021-11-11T20:37:45.333 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"ran_from_cephlab_playbook": true}, "changed": false} 2021-11-11T20:37:45.337 INFO:teuthology.task.ansible.out: PLAY [testnodes] *************************************************************** 2021-11-11T20:37:45.476 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:45 +0000 (0:00:00.176) 0:00:05.732 ***** 2021-11-11T20:37:45.495 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:45 +0000 (0:00:00.019) 0:00:05.751 ***** 2021-11-11T20:37:46.107 INFO:teuthology.task.ansible.out: TASK [Gathering Facts] ********************************************************* ok: [smithi096.front.sepia.ceph.com] 2021-11-11T20:37:46.136 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] 2021-11-11T20:37:46.165 INFO:teuthology.task.ansible.out:[WARNING]: Using run_once with the free strategy is not currently supported. This task will still be executed for every host in the inventory list. 2021-11-11T20:37:46.166 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:46 +0000 (0:00:00.669) 0:00:06.421 ***** 2021-11-11T20:37:46.231 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:46 +0000 (0:00:00.065) 0:00:06.486 ***** 2021-11-11T20:37:46.837 INFO:teuthology.task.ansible.out: TASK [users : Check keys_repo HEAD sha1] *************************************** 2021-11-11T20:37:46.839 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": "git ls-remote https://github.com/ceph/keys HEAD | awk '{ print $1 }'", "delta": "0:00:00.180263", "end": "2021-11-11 20:37:46.817104", "rc": 0, "start": "2021-11-11 20:37:46.636841", "stderr": "", "stderr_lines": [], "stdout": "d72e06d3c78c8d9639a8ca75effa043ee468910f", "stdout_lines": ["d72e06d3c78c8d9639a8ca75effa043ee468910f"]} 2021-11-11T20:37:46.859 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:46 +0000 (0:00:00.628) 0:00:07.115 ***** 2021-11-11T20:37:46.865 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": "git ls-remote https://github.com/ceph/keys HEAD | awk '{ print $1 }'", "delta": "0:00:00.190202", "end": "2021-11-11 20:37:46.821401", "rc": 0, "start": "2021-11-11 20:37:46.631199", "stderr": "", "stderr_lines": [], "stdout": "d72e06d3c78c8d9639a8ca75effa043ee468910f", "stdout_lines": ["d72e06d3c78c8d9639a8ca75effa043ee468910f"]} 2021-11-11T20:37:46.889 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:46 +0000 (0:00:00.030) 0:00:07.145 ***** 2021-11-11T20:37:47.050 INFO:teuthology.task.ansible.out: TASK [users : Check host's /keys-repo-sha1 sentinel file] ********************** 2021-11-11T20:37:47.052 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": ["cat", "/keys-repo-sha1"], "delta": "0:00:00.007770", "end": "2021-11-11 20:37:47.029658", "failed_when_result": false, "rc": 0, "start": "2021-11-11 20:37:47.021888", "stderr": "", "stderr_lines": [], "stdout": "b2a91f5f527dcc382f8ed49697328a42af627ee8", "stdout_lines": ["b2a91f5f527dcc382f8ed49697328a42af627ee8"]} 2021-11-11T20:37:47.072 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.183) 0:00:07.328 ***** 2021-11-11T20:37:47.090 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": ["cat", "/keys-repo-sha1"], "delta": "0:00:00.007851", "end": "2021-11-11 20:37:47.066906", "failed_when_result": false, "rc": 0, "start": "2021-11-11 20:37:47.059055", "stderr": "", "stderr_lines": [], "stdout": "b2a91f5f527dcc382f8ed49697328a42af627ee8", "stdout_lines": ["b2a91f5f527dcc382f8ed49697328a42af627ee8"]} 2021-11-11T20:37:47.115 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.042) 0:00:07.371 ***** 2021-11-11T20:37:47.141 INFO:teuthology.task.ansible.out: TASK [Determine if we can skip users and pubkeys updates] ********************** 2021-11-11T20:37:47.143 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:47.163 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.048) 0:00:07.420 ***** 2021-11-11T20:37:47.190 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:47.214 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.050) 0:00:07.470 ***** 2021-11-11T20:37:47.232 INFO:teuthology.task.ansible.out: TASK [Merge extra_admin_users into managed_admin_users] ************************ 2021-11-11T20:37:47.235 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:47.265 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.050) 0:00:07.520 ***** 2021-11-11T20:37:47.293 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:47.318 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.053) 0:00:07.574 ***** 2021-11-11T20:37:47.336 INFO:teuthology.task.ansible.out: TASK [Remove managed_admin_users from managed_users] *************************** 2021-11-11T20:37:47.338 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:47.370 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.051) 0:00:07.626 ***** 2021-11-11T20:37:47.399 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:47.422 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.052) 0:00:07.678 ***** 2021-11-11T20:37:47.441 INFO:teuthology.task.ansible.out: TASK [Filter the managed_users list] ******************************************* 2021-11-11T20:37:47.443 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:47.474 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.052) 0:00:07.730 ***** 2021-11-11T20:37:47.504 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:47.527 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.052) 0:00:07.783 ***** 2021-11-11T20:37:47.544 INFO:teuthology.task.ansible.out: TASK [Filter the managed_admin_users list] ************************************* 2021-11-11T20:37:47.546 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:47.567 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.040) 0:00:07.823 ***** 2021-11-11T20:37:47.594 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:37:47.617 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.049) 0:00:07.873 ***** 2021-11-11T20:37:47.792 INFO:teuthology.task.ansible.out: TASK [users : Set UID_MIN to 1001] ********************************************* 2021-11-11T20:37:47.794 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"backup": "", "changed": false, "msg": ""} 2021-11-11T20:37:47.815 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.198) 0:00:08.071 ***** 2021-11-11T20:37:47.853 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"backup": "", "changed": false, "msg": ""} 2021-11-11T20:37:47.876 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:37:47 +0000 (0:00:00.061) 0:00:08.132 ***** 2021-11-11T20:37:48.100 INFO:teuthology.task.ansible.out: TASK [Create all admin users with sudo access.] ******************************** 2021-11-11T20:37:48.102 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'andrewschoen', 'ovpn': 'andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1101, "groups": "sudo", "home": "/home/andrewschoen", "item": {"name": "andrewschoen", "ovpn": "andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9"}, "move_home": false, "name": "andrewschoen", "shell": "/bin/bash", "state": "present", "uid": 1101} 2021-11-11T20:37:48.154 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'andrewschoen', 'ovpn': 'andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1101, "groups": "sudo", "home": "/home/andrewschoen", "item": {"name": "andrewschoen", "ovpn": "andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9"}, "move_home": false, "name": "andrewschoen", "shell": "/bin/bash", "state": "present", "uid": 1101} 2021-11-11T20:37:48.289 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'zack', 'ovpn': 'zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1102, "groups": "sudo", "home": "/home/zack", "item": {"name": "zack", "ovpn": "zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n"}, "move_home": false, "name": "zack", "shell": "/bin/bash", "state": "present", "uid": 1102} 2021-11-11T20:37:48.353 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'zack', 'ovpn': 'zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1102, "groups": "sudo", "home": "/home/zack", "item": {"name": "zack", "ovpn": "zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n"}, "move_home": false, "name": "zack", "shell": "/bin/bash", "state": "present", "uid": 1102} 2021-11-11T20:37:48.478 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kdreyer', 'ovpn': 'kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1103, "groups": "sudo", "home": "/home/kdreyer", "item": {"name": "kdreyer", "ovpn": "kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd"}, "move_home": false, "name": "kdreyer", "shell": "/bin/bash", "state": "present", "uid": 1103} 2021-11-11T20:37:48.541 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kdreyer', 'ovpn': 'kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1103, "groups": "sudo", "home": "/home/kdreyer", "item": {"name": "kdreyer", "ovpn": "kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd"}, "move_home": false, "name": "kdreyer", "shell": "/bin/bash", "state": "present", "uid": 1103} 2021-11-11T20:37:48.676 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dmick', 'ovpn': 'dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1104, "groups": "sudo", "home": "/home/dmick", "item": {"name": "dmick", "ovpn": "dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n"}, "move_home": false, "name": "dmick", "shell": "/bin/bash", "state": "present", "uid": 1104} 2021-11-11T20:37:48.741 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dmick', 'ovpn': 'dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1104, "groups": "sudo", "home": "/home/dmick", "item": {"name": "dmick", "ovpn": "dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n"}, "move_home": false, "name": "dmick", "shell": "/bin/bash", "state": "present", "uid": 1104} 2021-11-11T20:37:48.866 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'sage', 'ovpn': 'sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1105, "groups": "sudo", "home": "/home/sage", "item": {"name": "sage", "ovpn": "sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n"}, "move_home": false, "name": "sage", "shell": "/bin/bash", "state": "present", "uid": 1105} 2021-11-11T20:37:48.929 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'sage', 'ovpn': 'sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1105, "groups": "sudo", "home": "/home/sage", "item": {"name": "sage", "ovpn": "sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n"}, "move_home": false, "name": "sage", "shell": "/bin/bash", "state": "present", "uid": 1105} 2021-11-11T20:37:49.065 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dgalloway', 'ovpn': 'dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1106, "groups": "sudo", "home": "/home/dgalloway", "item": {"name": "dgalloway", "ovpn": "dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n"}, "move_home": false, "name": "dgalloway", "shell": "/bin/bash", "state": "present", "uid": 1106} 2021-11-11T20:37:49.129 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dgalloway', 'ovpn': 'dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1106, "groups": "sudo", "home": "/home/dgalloway", "item": {"name": "dgalloway", "ovpn": "dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n"}, "move_home": false, "name": "dgalloway", "shell": "/bin/bash", "state": "present", "uid": 1106} 2021-11-11T20:37:49.254 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'akraitma', 'ovpn': 'akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1107, "groups": "sudo", "home": "/home/akraitma", "item": {"name": "akraitma", "ovpn": "akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045"}, "move_home": false, "name": "akraitma", "shell": "/bin/bash", "state": "present", "uid": 1107} 2021-11-11T20:37:49.317 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'akraitma', 'ovpn': 'akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1107, "groups": "sudo", "home": "/home/akraitma", "item": {"name": "akraitma", "ovpn": "akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045"}, "move_home": false, "name": "akraitma", "shell": "/bin/bash", "state": "present", "uid": 1107} 2021-11-11T20:37:49.454 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jcaratzas', 'ovpn': 'jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1108, "groups": "sudo", "home": "/home/jcaratzas", "item": {"name": "jcaratzas", "ovpn": "jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898"}, "move_home": false, "name": "jcaratzas", "shell": "/bin/bash", "state": "present", "uid": 1108} 2021-11-11T20:37:49.517 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jcaratzas', 'ovpn': 'jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1108, "groups": "sudo", "home": "/home/jcaratzas", "item": {"name": "jcaratzas", "ovpn": "jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898"}, "move_home": false, "name": "jcaratzas", "shell": "/bin/bash", "state": "present", "uid": 1108} 2021-11-11T20:37:49.652 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'trociny', 'ovpn': 'mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1109, "groups": "sudo", "home": "/home/trociny", "item": {"name": "trociny", "ovpn": "mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e"}, "move_home": false, "name": "trociny", "shell": "/bin/bash", "state": "present", "uid": 1109} 2021-11-11T20:37:49.715 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'trociny', 'ovpn': 'mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1109, "groups": "sudo", "home": "/home/trociny", "item": {"name": "trociny", "ovpn": "mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e"}, "move_home": false, "name": "trociny", "shell": "/bin/bash", "state": "present", "uid": 1109} 2021-11-11T20:37:49.841 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'smithfarm', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf', 'ovpn': 'smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1110, "groups": "sudo", "home": "/home/smithfarm", "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf", "name": "smithfarm", "ovpn": "smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007"}, "move_home": false, "name": "smithfarm", "shell": "/bin/bash", "state": "present", "uid": 1110} 2021-11-11T20:37:49.904 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'smithfarm', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf', 'ovpn': 'smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1110, "groups": "sudo", "home": "/home/smithfarm", "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf", "name": "smithfarm", "ovpn": "smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007"}, "move_home": false, "name": "smithfarm", "shell": "/bin/bash", "state": "present", "uid": 1110} 2021-11-11T20:37:50.041 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mbenjamin', 'ovpn': 'mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1111, "groups": "sudo", "home": "/home/mbenjamin", "item": {"name": "mbenjamin", "ovpn": "mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n"}, "move_home": false, "name": "mbenjamin", "shell": "/bin/bash", "state": "present", "uid": 1111} 2021-11-11T20:37:50.104 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mbenjamin', 'ovpn': 'mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1111, "groups": "sudo", "home": "/home/mbenjamin", "item": {"name": "mbenjamin", "ovpn": "mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n"}, "move_home": false, "name": "mbenjamin", "shell": "/bin/bash", "state": "present", "uid": 1111} 2021-11-11T20:37:50.228 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'aemerson', 'ovpn': 'aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1112, "groups": "sudo", "home": "/home/aemerson", "item": {"name": "aemerson", "ovpn": "aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n"}, "move_home": false, "name": "aemerson", "shell": "/bin/bash", "state": "present", "uid": 1112} 2021-11-11T20:37:50.301 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'aemerson', 'ovpn': 'aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1112, "groups": "sudo", "home": "/home/aemerson", "item": {"name": "aemerson", "ovpn": "aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n"}, "move_home": false, "name": "aemerson", "shell": "/bin/bash", "state": "present", "uid": 1112} 2021-11-11T20:37:50.426 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'amaredia', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour', 'ovpn': 'ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1113, "groups": "sudo", "home": "/home/amaredia", "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour", "name": "amaredia", "ovpn": "ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78"}, "move_home": false, "name": "amaredia", "shell": "/bin/bash", "state": "present", "uid": 1113} 2021-11-11T20:37:50.489 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'amaredia', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour', 'ovpn': 'ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1113, "groups": "sudo", "home": "/home/amaredia", "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour", "name": "amaredia", "ovpn": "ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78"}, "move_home": false, "name": "amaredia", "shell": "/bin/bash", "state": "present", "uid": 1113} 2021-11-11T20:37:50.624 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'tserlin', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet', 'ovpn': 'tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1114, "groups": "sudo", "home": "/home/tserlin", "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet", "name": "tserlin", "ovpn": "tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5"}, "move_home": false, "name": "tserlin", "shell": "/bin/bash", "state": "present", "uid": 1114} 2021-11-11T20:37:50.687 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'tserlin', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet', 'ovpn': 'tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1114, "groups": "sudo", "home": "/home/tserlin", "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet", "name": "tserlin", "ovpn": "tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5"}, "move_home": false, "name": "tserlin", "shell": "/bin/bash", "state": "present", "uid": 1114} 2021-11-11T20:37:50.812 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dis', 'ovpn': 'dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1115, "groups": "sudo", "home": "/home/dis", "item": {"name": "dis", "ovpn": "dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28"}, "move_home": false, "name": "dis", "shell": "/bin/bash", "state": "present", "uid": 1115} 2021-11-11T20:37:50.885 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dis', 'ovpn': 'dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1115, "groups": "sudo", "home": "/home/dis", "item": {"name": "dis", "ovpn": "dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28"}, "move_home": false, "name": "dis", "shell": "/bin/bash", "state": "present", "uid": 1115} 2021-11-11T20:37:51.010 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'gregf', 'ovpn': 'gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1116, "groups": "sudo", "home": "/home/gregf", "item": {"name": "gregf", "ovpn": "gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n"}, "move_home": false, "name": "gregf", "shell": "/bin/bash", "state": "present", "uid": 1116} 2021-11-11T20:37:51.074 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'gregf', 'ovpn': 'gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1116, "groups": "sudo", "home": "/home/gregf", "item": {"name": "gregf", "ovpn": "gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n"}, "move_home": false, "name": "gregf", "shell": "/bin/bash", "state": "present", "uid": 1116} 2021-11-11T20:37:51.198 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'joshd', 'ovpn': 'jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1117, "groups": "sudo", "home": "/home/joshd", "item": {"name": "joshd", "ovpn": "jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n"}, "move_home": false, "name": "joshd", "shell": "/bin/bash", "state": "present", "uid": 1117} 2021-11-11T20:37:51.271 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'joshd', 'ovpn': 'jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1117, "groups": "sudo", "home": "/home/joshd", "item": {"name": "joshd", "ovpn": "jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n"}, "move_home": false, "name": "joshd", "shell": "/bin/bash", "state": "present", "uid": 1117} 2021-11-11T20:37:51.396 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'cmeno', 'ovpn': 'cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1119, "groups": "sudo", "home": "/home/cmeno", "item": {"name": "cmeno", "ovpn": "cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed"}, "move_home": false, "name": "cmeno", "shell": "/bin/bash", "state": "present", "uid": 1119} 2021-11-11T20:37:51.459 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'cmeno', 'ovpn': 'cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1119, "groups": "sudo", "home": "/home/cmeno", "item": {"name": "cmeno", "ovpn": "cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed"}, "move_home": false, "name": "cmeno", "shell": "/bin/bash", "state": "present", "uid": 1119} 2021-11-11T20:37:51.595 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ivancich', 'ovpn': 'ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1120, "groups": "sudo", "home": "/home/ivancich", "item": {"name": "ivancich", "ovpn": "ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1"}, "move_home": false, "name": "ivancich", "shell": "/bin/bash", "state": "present", "uid": 1120} 2021-11-11T20:37:51.658 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ivancich', 'ovpn': 'ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1120, "groups": "sudo", "home": "/home/ivancich", "item": {"name": "ivancich", "ovpn": "ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1"}, "move_home": false, "name": "ivancich", "shell": "/bin/bash", "state": "present", "uid": 1120} 2021-11-11T20:37:51.783 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'yuriw', 'ovpn': 'yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1122, "groups": "sudo", "home": "/home/yuriw", "item": {"name": "yuriw", "ovpn": "yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n"}, "move_home": false, "name": "yuriw", "shell": "/bin/bash", "state": "present", "uid": 1122} 2021-11-11T20:37:51.846 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'yuriw', 'ovpn': 'yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1122, "groups": "sudo", "home": "/home/yuriw", "item": {"name": "yuriw", "ovpn": "yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n"}, "move_home": false, "name": "yuriw", "shell": "/bin/bash", "state": "present", "uid": 1122} 2021-11-11T20:37:51.974 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'tamil', 'ovpn': 'tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1123, "groups": "sudo", "home": "/home/tamil", "item": {"name": "tamil", "ovpn": "tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n"}, "move_home": false, "name": "tamil", "shell": "/bin/bash", "state": "present", "uid": 1123} 2021-11-11T20:37:52.037 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'tamil', 'ovpn': 'tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1123, "groups": "sudo", "home": "/home/tamil", "item": {"name": "tamil", "ovpn": "tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n"}, "move_home": false, "name": "tamil", "shell": "/bin/bash", "state": "present", "uid": 1123} 2021-11-11T20:37:52.173 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jowilkin', 'ovpn': 'jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1124, "groups": "sudo", "home": "/home/jowilkin", "item": {"name": "jowilkin", "ovpn": "jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n"}, "move_home": false, "name": "jowilkin", "shell": "/bin/bash", "state": "present", "uid": 1124} 2021-11-11T20:37:52.236 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jowilkin', 'ovpn': 'jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1124, "groups": "sudo", "home": "/home/jowilkin", "item": {"name": "jowilkin", "ovpn": "jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n"}, "move_home": false, "name": "jowilkin", "shell": "/bin/bash", "state": "present", "uid": 1124} 2021-11-11T20:37:52.372 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'bhubbard', 'ovpn': 'brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1125, "groups": "sudo", "home": "/home/bhubbard", "item": {"name": "bhubbard", "ovpn": "brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4"}, "move_home": false, "name": "bhubbard", "shell": "/bin/bash", "state": "present", "uid": 1125} 2021-11-11T20:37:52.424 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'bhubbard', 'ovpn': 'brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1125, "groups": "sudo", "home": "/home/bhubbard", "item": {"name": "bhubbard", "ovpn": "brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4"}, "move_home": false, "name": "bhubbard", "shell": "/bin/bash", "state": "present", "uid": 1125} 2021-11-11T20:37:52.559 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'yehudasa', 'ovpn': 'yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1126, "groups": "sudo", "home": "/home/yehudasa", "item": {"name": "yehudasa", "ovpn": "yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n"}, "move_home": false, "name": "yehudasa", "shell": "/bin/bash", "state": "present", "uid": 1126} 2021-11-11T20:37:52.623 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'yehudasa', 'ovpn': 'yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1126, "groups": "sudo", "home": "/home/yehudasa", "item": {"name": "yehudasa", "ovpn": "yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n"}, "move_home": false, "name": "yehudasa", "shell": "/bin/bash", "state": "present", "uid": 1126} 2021-11-11T20:37:52.758 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dang', 'ovpn': 'dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1127, "groups": "sudo", "home": "/home/dang", "item": {"name": "dang", "ovpn": "dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4"}, "move_home": false, "name": "dang", "shell": "/bin/bash", "state": "present", "uid": 1127} 2021-11-11T20:37:52.811 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dang', 'ovpn': 'dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1127, "groups": "sudo", "home": "/home/dang", "item": {"name": "dang", "ovpn": "dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4"}, "move_home": false, "name": "dang", "shell": "/bin/bash", "state": "present", "uid": 1127} 2021-11-11T20:37:52.951 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'branto', 'ovpn': 'branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1128, "groups": "sudo", "home": "/home/branto", "item": {"name": "branto", "ovpn": "branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb"}, "move_home": false, "name": "branto", "shell": "/bin/bash", "state": "present", "uid": 1128} 2021-11-11T20:37:53.004 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'branto', 'ovpn': 'branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1128, "groups": "sudo", "home": "/home/branto", "item": {"name": "branto", "ovpn": "branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb"}, "move_home": false, "name": "branto", "shell": "/bin/bash", "state": "present", "uid": 1128} 2021-11-11T20:37:53.149 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'xiaoxichen', 'ovpn': 'xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1129, "groups": "sudo", "home": "/home/xiaoxichen", "item": {"name": "xiaoxichen", "ovpn": "xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6"}, "move_home": false, "name": "xiaoxichen", "shell": "/bin/bash", "state": "present", "uid": 1129} 2021-11-11T20:37:53.202 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'xiaoxichen', 'ovpn': 'xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1129, "groups": "sudo", "home": "/home/xiaoxichen", "item": {"name": "xiaoxichen", "ovpn": "xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6"}, "move_home": false, "name": "xiaoxichen", "shell": "/bin/bash", "state": "present", "uid": 1129} 2021-11-11T20:37:53.348 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ffilz', 'ovpn': 'ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1130, "groups": "sudo", "home": "/home/ffilz", "item": {"name": "ffilz", "ovpn": "ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e"}, "move_home": false, "name": "ffilz", "shell": "/bin/bash", "state": "present", "uid": 1130} 2021-11-11T20:37:53.401 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ffilz', 'ovpn': 'ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1130, "groups": "sudo", "home": "/home/ffilz", "item": {"name": "ffilz", "ovpn": "ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e"}, "move_home": false, "name": "ffilz", "shell": "/bin/bash", "state": "present", "uid": 1130} 2021-11-11T20:37:53.537 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'joao', 'ovpn': 'joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1131, "groups": "sudo", "home": "/home/joao", "item": {"name": "joao", "ovpn": "joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n"}, "move_home": false, "name": "joao", "shell": "/bin/bash", "state": "present", "uid": 1131} 2021-11-11T20:37:53.590 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'joao', 'ovpn': 'joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1131, "groups": "sudo", "home": "/home/joao", "item": {"name": "joao", "ovpn": "joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n"}, "move_home": false, "name": "joao", "shell": "/bin/bash", "state": "present", "uid": 1131} 2021-11-11T20:37:53.725 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'nhm', 'ovpn': 'nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1132, "groups": "sudo", "home": "/home/nhm", "item": {"name": "nhm", "ovpn": "nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n"}, "move_home": false, "name": "nhm", "shell": "/bin/bash", "state": "present", "uid": 1132} 2021-11-11T20:37:53.778 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'nhm', 'ovpn': 'nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1132, "groups": "sudo", "home": "/home/nhm", "item": {"name": "nhm", "ovpn": "nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n"}, "move_home": false, "name": "nhm", "shell": "/bin/bash", "state": "present", "uid": 1132} 2021-11-11T20:37:53.924 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mkidd', 'ovpn': 'linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1134, "groups": "sudo", "home": "/home/mkidd", "item": {"name": "mkidd", "ovpn": "linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22"}, "move_home": false, "name": "mkidd", "shell": "/bin/bash", "state": "present", "uid": 1134} 2021-11-11T20:37:53.977 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mkidd', 'ovpn': 'linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1134, "groups": "sudo", "home": "/home/mkidd", "item": {"name": "mkidd", "ovpn": "linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22"}, "move_home": false, "name": "mkidd", "shell": "/bin/bash", "state": "present", "uid": 1134} 2021-11-11T20:37:54.122 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jlopez', 'ovpn': 'JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1135, "groups": "sudo", "home": "/home/jlopez", "item": {"name": "jlopez", "ovpn": "JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af"}, "move_home": false, "name": "jlopez", "shell": "/bin/bash", "state": "present", "uid": 1135} 2021-11-11T20:37:54.175 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jlopez', 'ovpn': 'JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1135, "groups": "sudo", "home": "/home/jlopez", "item": {"name": "jlopez", "ovpn": "JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af"}, "move_home": false, "name": "jlopez", "shell": "/bin/bash", "state": "present", "uid": 1135} 2021-11-11T20:37:54.311 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'haomaiwang', 'ovpn': 'yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1136, "groups": "sudo", "home": "/home/haomaiwang", "item": {"name": "haomaiwang", "ovpn": "yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb"}, "move_home": false, "name": "haomaiwang", "shell": "/bin/bash", "state": "present", "uid": 1136} 2021-11-11T20:37:54.363 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'haomaiwang', 'ovpn': 'yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1136, "groups": "sudo", "home": "/home/haomaiwang", "item": {"name": "haomaiwang", "ovpn": "yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb"}, "move_home": false, "name": "haomaiwang", "shell": "/bin/bash", "state": "present", "uid": 1136} 2021-11-11T20:37:54.510 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kchai', 'ovpn': 'kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1137, "groups": "sudo", "home": "/home/kchai", "item": {"name": "kchai", "ovpn": "kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3"}, "move_home": false, "name": "kchai", "shell": "/bin/bash", "state": "present", "uid": 1137} 2021-11-11T20:37:54.563 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kchai', 'ovpn': 'kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1137, "groups": "sudo", "home": "/home/kchai", "item": {"name": "kchai", "ovpn": "kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3"}, "move_home": false, "name": "kchai", "shell": "/bin/bash", "state": "present", "uid": 1137} 2021-11-11T20:37:54.699 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1138, "groups": "sudo", "home": "/home/vumrao", "item": {"name": "vumrao", "ovpn": "vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2"}, "move_home": false, "name": "vumrao", "shell": "/bin/bash", "state": "present", "uid": 1138} 2021-11-11T20:37:54.751 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1138, "groups": "sudo", "home": "/home/vumrao", "item": {"name": "vumrao", "ovpn": "vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2"}, "move_home": false, "name": "vumrao", "shell": "/bin/bash", "state": "present", "uid": 1138} 2021-11-11T20:37:54.897 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dfuller', 'ovpn': 'dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1139, "groups": "sudo", "home": "/home/dfuller", "item": {"name": "dfuller", "ovpn": "dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n"}, "move_home": false, "name": "dfuller", "shell": "/bin/bash", "state": "present", "uid": 1139} 2021-11-11T20:37:54.949 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dfuller', 'ovpn': 'dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1139, "groups": "sudo", "home": "/home/dfuller", "item": {"name": "dfuller", "ovpn": "dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n"}, "move_home": false, "name": "dfuller", "shell": "/bin/bash", "state": "present", "uid": 1139} 2021-11-11T20:37:55.096 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'owasserm', 'ovpn': 'owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1140, "groups": "sudo", "home": "/home/owasserm", "item": {"name": "owasserm", "ovpn": "owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n"}, "move_home": false, "name": "owasserm", "shell": "/bin/bash", "state": "present", "uid": 1140} 2021-11-11T20:37:55.148 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'owasserm', 'ovpn': 'owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1140, "groups": "sudo", "home": "/home/owasserm", "item": {"name": "owasserm", "ovpn": "owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n"}, "move_home": false, "name": "owasserm", "shell": "/bin/bash", "state": "present", "uid": 1140} 2021-11-11T20:37:55.283 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vasu', 'ovpn': 'vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1141, "groups": "sudo", "home": "/home/vasu", "item": {"name": "vasu", "ovpn": "vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n"}, "move_home": false, "name": "vasu", "shell": "/bin/bash", "state": "present", "uid": 1141} 2021-11-11T20:37:55.336 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vasu', 'ovpn': 'vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1141, "groups": "sudo", "home": "/home/vasu", "item": {"name": "vasu", "ovpn": "vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n"}, "move_home": false, "name": "vasu", "shell": "/bin/bash", "state": "present", "uid": 1141} 2021-11-11T20:37:55.481 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'smohan', 'ovpn': 'shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1142, "groups": "sudo", "home": "/home/smohan", "item": {"name": "smohan", "ovpn": "shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37"}, "move_home": false, "name": "smohan", "shell": "/bin/bash", "state": "present", "uid": 1142} 2021-11-11T20:37:55.534 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'smohan', 'ovpn': 'shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1142, "groups": "sudo", "home": "/home/smohan", "item": {"name": "smohan", "ovpn": "shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37"}, "move_home": false, "name": "smohan", "shell": "/bin/bash", "state": "present", "uid": 1142} 2021-11-11T20:37:55.669 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'abhi', 'ovpn': 'abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1143, "groups": "sudo", "home": "/home/abhi", "item": {"name": "abhi", "ovpn": "abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114"}, "move_home": false, "name": "abhi", "shell": "/bin/bash", "state": "present", "uid": 1143} 2021-11-11T20:37:55.722 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'abhi', 'ovpn': 'abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1143, "groups": "sudo", "home": "/home/abhi", "item": {"name": "abhi", "ovpn": "abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114"}, "move_home": false, "name": "abhi", "shell": "/bin/bash", "state": "present", "uid": 1143} 2021-11-11T20:37:55.867 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'cbodley', 'ovpn': 'cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1144, "groups": "sudo", "home": "/home/cbodley", "item": {"name": "cbodley", "ovpn": "cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611"}, "move_home": false, "name": "cbodley", "shell": "/bin/bash", "state": "present", "uid": 1144} 2021-11-11T20:37:55.920 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'cbodley', 'ovpn': 'cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1144, "groups": "sudo", "home": "/home/cbodley", "item": {"name": "cbodley", "ovpn": "cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611"}, "move_home": false, "name": "cbodley", "shell": "/bin/bash", "state": "present", "uid": 1144} 2021-11-11T20:37:56.057 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mwatts', 'ovpn': 'mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1145, "groups": "sudo", "home": "/home/mwatts", "item": {"name": "mwatts", "ovpn": "mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195"}, "move_home": false, "name": "mwatts", "shell": "/bin/bash", "state": "present", "uid": 1145} 2021-11-11T20:37:56.110 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mwatts', 'ovpn': 'mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1145, "groups": "sudo", "home": "/home/mwatts", "item": {"name": "mwatts", "ovpn": "mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195"}, "move_home": false, "name": "mwatts", "shell": "/bin/bash", "state": "present", "uid": 1145} 2021-11-11T20:37:56.256 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1146, "groups": "sudo", "home": "/home/prsrivas", "item": {"name": "prsrivas", "ovpn": "pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n"}, "move_home": false, "name": "prsrivas", "shell": "/bin/bash", "state": "present", "uid": 1146} 2021-11-11T20:37:56.308 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1146, "groups": "sudo", "home": "/home/prsrivas", "item": {"name": "prsrivas", "ovpn": "pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n"}, "move_home": false, "name": "prsrivas", "shell": "/bin/bash", "state": "present", "uid": 1146} 2021-11-11T20:37:56.444 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'pdonnell', 'ovpn': 'pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1147, "groups": "sudo", "home": "/home/pdonnell", "item": {"name": "pdonnell", "ovpn": "pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea"}, "move_home": false, "name": "pdonnell", "shell": "/bin/bash", "state": "present", "uid": 1147} 2021-11-11T20:37:56.497 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'pdonnell', 'ovpn': 'pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1147, "groups": "sudo", "home": "/home/pdonnell", "item": {"name": "pdonnell", "ovpn": "pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea"}, "move_home": false, "name": "pdonnell", "shell": "/bin/bash", "state": "present", "uid": 1147} 2021-11-11T20:37:56.643 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jlayton', 'ovpn': 'jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1148, "groups": "sudo", "home": "/home/jlayton", "item": {"name": "jlayton", "ovpn": "jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868"}, "move_home": false, "name": "jlayton", "shell": "/bin/bash", "state": "present", "uid": 1148} 2021-11-11T20:37:56.686 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jlayton', 'ovpn': 'jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1148, "groups": "sudo", "home": "/home/jlayton", "item": {"name": "jlayton", "ovpn": "jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868"}, "move_home": false, "name": "jlayton", "shell": "/bin/bash", "state": "present", "uid": 1148} 2021-11-11T20:37:56.832 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rzarzynski', 'ovpn': 'rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1149, "groups": "sudo", "home": "/home/rzarzynski", "item": {"name": "rzarzynski", "ovpn": "rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f"}, "move_home": false, "name": "rzarzynski", "shell": "/bin/bash", "state": "present", "uid": 1149} 2021-11-11T20:37:56.885 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rzarzynski', 'ovpn': 'rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1149, "groups": "sudo", "home": "/home/rzarzynski", "item": {"name": "rzarzynski", "ovpn": "rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f"}, "move_home": false, "name": "rzarzynski", "shell": "/bin/bash", "state": "present", "uid": 1149} 2021-11-11T20:37:57.031 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rdias', 'ovpn': 'rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1150, "groups": "sudo", "home": "/home/rdias", "item": {"name": "rdias", "ovpn": "rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e"}, "move_home": false, "name": "rdias", "shell": "/bin/bash", "state": "present", "uid": 1150} 2021-11-11T20:37:57.074 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rdias', 'ovpn': 'rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1150, "groups": "sudo", "home": "/home/rdias", "item": {"name": "rdias", "ovpn": "rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e"}, "move_home": false, "name": "rdias", "shell": "/bin/bash", "state": "present", "uid": 1150} 2021-11-11T20:37:57.220 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vshankar', 'ovpn': 'vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1151, "groups": "sudo", "home": "/home/vshankar", "item": {"name": "vshankar", "ovpn": "vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20"}, "move_home": false, "name": "vshankar", "shell": "/bin/bash", "state": "present", "uid": 1151} 2021-11-11T20:37:57.272 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vshankar', 'ovpn': 'vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1151, "groups": "sudo", "home": "/home/vshankar", "item": {"name": "vshankar", "ovpn": "vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20"}, "move_home": false, "name": "vshankar", "shell": "/bin/bash", "state": "present", "uid": 1151} 2021-11-11T20:37:57.419 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'akupczyk', 'ovpn': 'adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1152, "groups": "sudo", "home": "/home/akupczyk", "item": {"name": "akupczyk", "ovpn": "adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58"}, "move_home": false, "name": "akupczyk", "shell": "/bin/bash", "state": "present", "uid": 1152} 2021-11-11T20:37:57.462 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'akupczyk', 'ovpn': 'adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1152, "groups": "sudo", "home": "/home/akupczyk", "item": {"name": "akupczyk", "ovpn": "adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58"}, "move_home": false, "name": "akupczyk", "shell": "/bin/bash", "state": "present", "uid": 1152} 2021-11-11T20:37:57.618 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'nojha', 'ovpn': 'nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1153, "groups": "sudo", "home": "/home/nojha", "item": {"name": "nojha", "ovpn": "nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n"}, "move_home": false, "name": "nojha", "shell": "/bin/bash", "state": "present", "uid": 1153} 2021-11-11T20:37:57.660 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'nojha', 'ovpn': 'nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1153, "groups": "sudo", "home": "/home/nojha", "item": {"name": "nojha", "ovpn": "nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n"}, "move_home": false, "name": "nojha", "shell": "/bin/bash", "state": "present", "uid": 1153} 2021-11-11T20:37:57.806 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ifed01', 'ovpn': 'ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1154, "groups": "sudo", "home": "/home/ifed01", "item": {"name": "ifed01", "ovpn": "ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56"}, "move_home": false, "name": "ifed01", "shell": "/bin/bash", "state": "present", "uid": 1154} 2021-11-11T20:37:57.848 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ifed01', 'ovpn': 'ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1154, "groups": "sudo", "home": "/home/ifed01", "item": {"name": "ifed01", "ovpn": "ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56"}, "move_home": false, "name": "ifed01", "shell": "/bin/bash", "state": "present", "uid": 1154} 2021-11-11T20:37:58.007 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'myoungwon', 'ovpn': 'myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1155, "groups": "sudo", "home": "/home/myoungwon", "item": {"name": "myoungwon", "ovpn": "myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71"}, "move_home": false, "name": "myoungwon", "shell": "/bin/bash", "state": "present", "uid": 1155} 2021-11-11T20:37:58.049 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'myoungwon', 'ovpn': 'myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1155, "groups": "sudo", "home": "/home/myoungwon", "item": {"name": "myoungwon", "ovpn": "myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71"}, "move_home": false, "name": "myoungwon", "shell": "/bin/bash", "state": "present", "uid": 1155} 2021-11-11T20:37:58.196 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'gabrioux', 'ovpn': 'gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1156, "groups": "sudo", "home": "/home/gabrioux", "item": {"name": "gabrioux", "ovpn": "gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2"}, "move_home": false, "name": "gabrioux", "shell": "/bin/bash", "state": "present", "uid": 1156} 2021-11-11T20:37:58.248 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'gabrioux', 'ovpn': 'gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1156, "groups": "sudo", "home": "/home/gabrioux", "item": {"name": "gabrioux", "ovpn": "gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2"}, "move_home": false, "name": "gabrioux", "shell": "/bin/bash", "state": "present", "uid": 1156} 2021-11-11T20:37:58.395 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'leseb', 'ovpn': 'leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1157, "groups": "sudo", "home": "/home/leseb", "item": {"name": "leseb", "ovpn": "leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f"}, "move_home": false, "name": "leseb", "shell": "/bin/bash", "state": "present", "uid": 1157} 2021-11-11T20:37:58.437 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'leseb', 'ovpn': 'leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1157, "groups": "sudo", "home": "/home/leseb", "item": {"name": "leseb", "ovpn": "leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f"}, "move_home": false, "name": "leseb", "shell": "/bin/bash", "state": "present", "uid": 1157} 2021-11-11T20:37:58.583 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'hchen', 'ovpn': 'hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1158, "groups": "sudo", "home": "/home/hchen", "item": {"name": "hchen", "ovpn": "hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca"}, "move_home": false, "name": "hchen", "shell": "/bin/bash", "state": "present", "uid": 1158} 2021-11-11T20:37:58.636 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'hchen', 'ovpn': 'hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1158, "groups": "sudo", "home": "/home/hchen", "item": {"name": "hchen", "ovpn": "hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca"}, "move_home": false, "name": "hchen", "shell": "/bin/bash", "state": "present", "uid": 1158} 2021-11-11T20:37:58.781 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jcollin', 'ovpn': 'jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1159, "groups": "sudo", "home": "/home/jcollin", "item": {"name": "jcollin", "ovpn": "jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n"}, "move_home": false, "name": "jcollin", "shell": "/bin/bash", "state": "present", "uid": 1159} 2021-11-11T20:37:58.823 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jcollin', 'ovpn': 'jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1159, "groups": "sudo", "home": "/home/jcollin", "item": {"name": "jcollin", "ovpn": "jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n"}, "move_home": false, "name": "jcollin", "shell": "/bin/bash", "state": "present", "uid": 1159} 2021-11-11T20:37:58.969 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'xxg', 'ovpn': 'xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1160, "groups": "sudo", "home": "/home/xxg", "item": {"name": "xxg", "ovpn": "xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc"}, "move_home": false, "name": "xxg", "shell": "/bin/bash", "state": "present", "uid": 1160} 2021-11-11T20:37:59.022 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'xxg', 'ovpn': 'xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1160, "groups": "sudo", "home": "/home/xxg", "item": {"name": "xxg", "ovpn": "xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc"}, "move_home": false, "name": "xxg", "shell": "/bin/bash", "state": "present", "uid": 1160} 2021-11-11T20:37:59.169 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'pcuzner', 'ovpn': 'pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1161, "groups": "sudo", "home": "/home/pcuzner", "item": {"name": "pcuzner", "ovpn": "pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53"}, "move_home": false, "name": "pcuzner", "shell": "/bin/bash", "state": "present", "uid": 1161} 2021-11-11T20:37:59.211 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'pcuzner', 'ovpn': 'pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1161, "groups": "sudo", "home": "/home/pcuzner", "item": {"name": "pcuzner", "ovpn": "pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53"}, "move_home": false, "name": "pcuzner", "shell": "/bin/bash", "state": "present", "uid": 1161} 2021-11-11T20:37:59.357 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mkogan', 'ovpn': 'mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1162, "groups": "sudo", "home": "/home/mkogan", "item": {"name": "mkogan", "ovpn": "mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2"}, "move_home": false, "name": "mkogan", "shell": "/bin/bash", "state": "present", "uid": 1162} 2021-11-11T20:37:59.412 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mkogan', 'ovpn': 'mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1162, "groups": "sudo", "home": "/home/mkogan", "item": {"name": "mkogan", "ovpn": "mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2"}, "move_home": false, "name": "mkogan", "shell": "/bin/bash", "state": "present", "uid": 1162} 2021-11-11T20:37:59.548 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1138, "groups": "sudo", "home": "/home/vumrao", "item": {"name": "vumrao", "ovpn": "vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed"}, "move_home": false, "name": "vumrao", "shell": "/bin/bash", "state": "present", "uid": 1138} 2021-11-11T20:37:59.632 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1138, "groups": "sudo", "home": "/home/vumrao", "item": {"name": "vumrao", "ovpn": "vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed"}, "move_home": false, "name": "vumrao", "shell": "/bin/bash", "state": "present", "uid": 1138} 2021-11-11T20:37:59.747 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'henrix', 'ovpn': 'henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1163, "groups": "sudo", "home": "/home/henrix", "item": {"name": "henrix", "ovpn": "henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736"}, "move_home": false, "name": "henrix", "shell": "/bin/bash", "state": "present", "uid": 1163} 2021-11-11T20:37:59.820 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'henrix', 'ovpn': 'henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1163, "groups": "sudo", "home": "/home/henrix", "item": {"name": "henrix", "ovpn": "henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736"}, "move_home": false, "name": "henrix", "shell": "/bin/bash", "state": "present", "uid": 1163} 2021-11-11T20:37:59.945 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'pbs1108', 'ovpn': 'pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1164, "groups": "sudo", "home": "/home/pbs1108", "item": {"name": "pbs1108", "ovpn": "pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26"}, "move_home": false, "name": "pbs1108", "shell": "/bin/bash", "state": "present", "uid": 1164} 2021-11-11T20:38:00.020 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'pbs1108', 'ovpn': 'pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1164, "groups": "sudo", "home": "/home/pbs1108", "item": {"name": "pbs1108", "ovpn": "pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26"}, "move_home": false, "name": "pbs1108", "shell": "/bin/bash", "state": "present", "uid": 1164} 2021-11-11T20:38:00.144 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'clacroix', 'ovpn': 'clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1165, "groups": "sudo", "home": "/home/clacroix", "item": {"name": "clacroix", "ovpn": "clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315"}, "move_home": false, "name": "clacroix", "shell": "/bin/bash", "state": "present", "uid": 1165} 2021-11-11T20:38:00.217 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'clacroix', 'ovpn': 'clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1165, "groups": "sudo", "home": "/home/clacroix", "item": {"name": "clacroix", "ovpn": "clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315"}, "move_home": false, "name": "clacroix", "shell": "/bin/bash", "state": "present", "uid": 1165} 2021-11-11T20:38:00.331 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'epuertat', 'ovpn': 'epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1166, "groups": "sudo", "home": "/home/epuertat", "item": {"name": "epuertat", "ovpn": "epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006"}, "move_home": false, "name": "epuertat", "shell": "/bin/bash", "state": "present", "uid": 1166} 2021-11-11T20:38:00.404 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'epuertat', 'ovpn': 'epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1166, "groups": "sudo", "home": "/home/epuertat", "item": {"name": "epuertat", "ovpn": "epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006"}, "move_home": false, "name": "epuertat", "shell": "/bin/bash", "state": "present", "uid": 1166} 2021-11-11T20:38:00.530 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'tdehler', 'ovpn': 'tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1167, "groups": "sudo", "home": "/home/tdehler", "item": {"name": "tdehler", "ovpn": "tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461"}, "move_home": false, "name": "tdehler", "shell": "/bin/bash", "state": "present", "uid": 1167} 2021-11-11T20:38:00.604 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'tdehler', 'ovpn': 'tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1167, "groups": "sudo", "home": "/home/tdehler", "item": {"name": "tdehler", "ovpn": "tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461"}, "move_home": false, "name": "tdehler", "shell": "/bin/bash", "state": "present", "uid": 1167} 2021-11-11T20:38:00.728 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'laura', 'ovpn': 'laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1168, "groups": "sudo", "home": "/home/laura", "item": {"name": "laura", "ovpn": "laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305"}, "move_home": false, "name": "laura", "shell": "/bin/bash", "state": "present", "uid": 1168} 2021-11-11T20:38:00.802 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'laura', 'ovpn': 'laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1168, "groups": "sudo", "home": "/home/laura", "item": {"name": "laura", "ovpn": "laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305"}, "move_home": false, "name": "laura", "shell": "/bin/bash", "state": "present", "uid": 1168} 2021-11-11T20:38:00.918 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'yaarit', 'ovpn': 'yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1169, "groups": "sudo", "home": "/home/yaarit", "item": {"name": "yaarit", "ovpn": "yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce"}, "move_home": false, "name": "yaarit", "shell": "/bin/bash", "state": "present", "uid": 1169} 2021-11-11T20:38:01.002 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'yaarit', 'ovpn': 'yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1169, "groups": "sudo", "home": "/home/yaarit", "item": {"name": "yaarit", "ovpn": "yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce"}, "move_home": false, "name": "yaarit", "shell": "/bin/bash", "state": "present", "uid": 1169} 2021-11-11T20:38:01.116 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rpavani1998', 'ovpn': 'admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1170, "groups": "sudo", "home": "/home/rpavani1998", "item": {"name": "rpavani1998", "ovpn": "admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3"}, "move_home": false, "name": "rpavani1998", "shell": "/bin/bash", "state": "present", "uid": 1170} 2021-11-11T20:38:01.190 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rpavani1998', 'ovpn': 'admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1170, "groups": "sudo", "home": "/home/rpavani1998", "item": {"name": "rpavani1998", "ovpn": "admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3"}, "move_home": false, "name": "rpavani1998", "shell": "/bin/bash", "state": "present", "uid": 1170} 2021-11-11T20:38:01.315 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rishabh', 'ovpn': 'rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1171, "groups": "sudo", "home": "/home/rishabh", "item": {"name": "rishabh", "ovpn": "rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4"}, "move_home": false, "name": "rishabh", "shell": "/bin/bash", "state": "present", "uid": 1171} 2021-11-11T20:38:01.388 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rishabh', 'ovpn': 'rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1171, "groups": "sudo", "home": "/home/rishabh", "item": {"name": "rishabh", "ovpn": "rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4"}, "move_home": false, "name": "rishabh", "shell": "/bin/bash", "state": "present", "uid": 1171} 2021-11-11T20:38:01.503 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'skrah', 'ovpn': 'skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1172, "groups": "sudo", "home": "/home/skrah", "item": {"name": "skrah", "ovpn": "skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2"}, "move_home": false, "name": "skrah", "shell": "/bin/bash", "state": "present", "uid": 1172} 2021-11-11T20:38:01.587 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'skrah', 'ovpn': 'skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1172, "groups": "sudo", "home": "/home/skrah", "item": {"name": "skrah", "ovpn": "skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2"}, "move_home": false, "name": "skrah", "shell": "/bin/bash", "state": "present", "uid": 1172} 2021-11-11T20:38:01.701 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'smanjara', 'ovpn': 'smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1173, "groups": "sudo", "home": "/home/smanjara", "item": {"name": "smanjara", "ovpn": "smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f"}, "move_home": false, "name": "smanjara", "shell": "/bin/bash", "state": "present", "uid": 1173} 2021-11-11T20:38:01.775 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'smanjara', 'ovpn': 'smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1173, "groups": "sudo", "home": "/home/smanjara", "item": {"name": "smanjara", "ovpn": "smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f"}, "move_home": false, "name": "smanjara", "shell": "/bin/bash", "state": "present", "uid": 1173} 2021-11-11T20:38:01.889 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'bengland', 'ovpn': 'bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1174, "groups": "sudo", "home": "/home/bengland", "item": {"name": "bengland", "ovpn": "bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41"}, "move_home": false, "name": "bengland", "shell": "/bin/bash", "state": "present", "uid": 1174} 2021-11-11T20:38:01.973 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'bengland', 'ovpn': 'bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1174, "groups": "sudo", "home": "/home/bengland", "item": {"name": "bengland", "ovpn": "bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41"}, "move_home": false, "name": "bengland", "shell": "/bin/bash", "state": "present", "uid": 1174} 2021-11-11T20:38:02.087 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'pnawracay', 'ovpn': 'pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1175, "groups": "sudo", "home": "/home/pnawracay", "item": {"name": "pnawracay", "ovpn": "pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d"}, "move_home": false, "name": "pnawracay", "shell": "/bin/bash", "state": "present", "uid": 1175} 2021-11-11T20:38:02.161 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'pnawracay', 'ovpn': 'pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1175, "groups": "sudo", "home": "/home/pnawracay", "item": {"name": "pnawracay", "ovpn": "pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d"}, "move_home": false, "name": "pnawracay", "shell": "/bin/bash", "state": "present", "uid": 1175} 2021-11-11T20:38:02.286 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'alfonsomthd', 'ovpn': 'alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1176, "groups": "sudo", "home": "/home/alfonsomthd", "item": {"name": "alfonsomthd", "ovpn": "alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47"}, "move_home": false, "name": "alfonsomthd", "shell": "/bin/bash", "state": "present", "uid": 1176} 2021-11-11T20:38:02.359 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'alfonsomthd', 'ovpn': 'alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1176, "groups": "sudo", "home": "/home/alfonsomthd", "item": {"name": "alfonsomthd", "ovpn": "alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47"}, "move_home": false, "name": "alfonsomthd", "shell": "/bin/bash", "state": "present", "uid": 1176} 2021-11-11T20:38:02.485 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'oliveiradan', 'ovpn': 'oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1177, "groups": "sudo", "home": "/home/oliveiradan", "item": {"name": "oliveiradan", "ovpn": "oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e"}, "move_home": false, "name": "oliveiradan", "shell": "/bin/bash", "state": "present", "uid": 1177} 2021-11-11T20:38:02.548 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'oliveiradan', 'ovpn': 'oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1177, "groups": "sudo", "home": "/home/oliveiradan", "item": {"name": "oliveiradan", "ovpn": "oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e"}, "move_home": false, "name": "oliveiradan", "shell": "/bin/bash", "state": "present", "uid": 1177} 2021-11-11T20:38:02.673 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'swagner', 'ovpn': 'swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1178, "groups": "sudo", "home": "/home/swagner", "item": {"name": "swagner", "ovpn": "swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba"}, "move_home": false, "name": "swagner", "shell": "/bin/bash", "state": "present", "uid": 1178} 2021-11-11T20:38:02.747 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'swagner', 'ovpn': 'swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1178, "groups": "sudo", "home": "/home/swagner", "item": {"name": "swagner", "ovpn": "swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba"}, "move_home": false, "name": "swagner", "shell": "/bin/bash", "state": "present", "uid": 1178} 2021-11-11T20:38:02.872 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'yuvalif', 'ovpn': 'ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1179, "groups": "sudo", "home": "/home/yuvalif", "item": {"name": "yuvalif", "ovpn": "ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n"}, "move_home": false, "name": "yuvalif", "shell": "/bin/bash", "state": "present", "uid": 1179} 2021-11-11T20:38:02.936 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'yuvalif', 'ovpn': 'ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1179, "groups": "sudo", "home": "/home/yuvalif", "item": {"name": "yuvalif", "ovpn": "ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n"}, "move_home": false, "name": "yuvalif", "shell": "/bin/bash", "state": "present", "uid": 1179} 2021-11-11T20:38:03.075 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kkeithle', 'ovpn': 'kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1180, "groups": "sudo", "home": "/home/kkeithle", "item": {"name": "kkeithle", "ovpn": "kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852"}, "move_home": false, "name": "kkeithle", "shell": "/bin/bash", "state": "present", "uid": 1180} 2021-11-11T20:38:03.138 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kkeithle', 'ovpn': 'kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1180, "groups": "sudo", "home": "/home/kkeithle", "item": {"name": "kkeithle", "ovpn": "kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852"}, "move_home": false, "name": "kkeithle", "shell": "/bin/bash", "state": "present", "uid": 1180} 2021-11-11T20:38:03.274 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'emmericp', 'ovpn': 'paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1181, "groups": "sudo", "home": "/home/emmericp", "item": {"name": "emmericp", "ovpn": "paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130"}, "move_home": false, "name": "emmericp", "shell": "/bin/bash", "state": "present", "uid": 1181} 2021-11-11T20:38:03.327 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'emmericp', 'ovpn': 'paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1181, "groups": "sudo", "home": "/home/emmericp", "item": {"name": "emmericp", "ovpn": "paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130"}, "move_home": false, "name": "emmericp", "shell": "/bin/bash", "state": "present", "uid": 1181} 2021-11-11T20:38:03.462 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mchangir', 'ovpn': 'mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1182, "groups": "sudo", "home": "/home/mchangir", "item": {"name": "mchangir", "ovpn": "mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0"}, "move_home": false, "name": "mchangir", "shell": "/bin/bash", "state": "present", "uid": 1182} 2021-11-11T20:38:03.525 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mchangir', 'ovpn': 'mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1182, "groups": "sudo", "home": "/home/mchangir", "item": {"name": "mchangir", "ovpn": "mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0"}, "move_home": false, "name": "mchangir", "shell": "/bin/bash", "state": "present", "uid": 1182} 2021-11-11T20:38:03.661 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'sjust', 'ovpn': 'sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1185, "groups": "sudo", "home": "/home/sjust", "item": {"name": "sjust", "ovpn": "sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n"}, "move_home": false, "name": "sjust", "shell": "/bin/bash", "state": "present", "uid": 1185} 2021-11-11T20:38:03.714 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'sjust', 'ovpn': 'sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1185, "groups": "sudo", "home": "/home/sjust", "item": {"name": "sjust", "ovpn": "sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n"}, "move_home": false, "name": "sjust", "shell": "/bin/bash", "state": "present", "uid": 1185} 2021-11-11T20:38:03.859 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ideepika', 'ovpn': 'deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1186, "groups": "sudo", "home": "/home/ideepika", "item": {"name": "ideepika", "ovpn": "deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f"}, "move_home": false, "name": "ideepika", "shell": "/bin/bash", "state": "present", "uid": 1186} 2021-11-11T20:38:03.912 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ideepika', 'ovpn': 'deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1186, "groups": "sudo", "home": "/home/ideepika", "item": {"name": "ideepika", "ovpn": "deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f"}, "move_home": false, "name": "ideepika", "shell": "/bin/bash", "state": "present", "uid": 1186} 2021-11-11T20:38:04.049 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'gsalomon', 'ovpn': 'gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1187, "groups": "sudo", "home": "/home/gsalomon", "item": {"name": "gsalomon", "ovpn": "gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46"}, "move_home": false, "name": "gsalomon", "shell": "/bin/bash", "state": "present", "uid": 1187} 2021-11-11T20:38:04.101 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'gsalomon', 'ovpn': 'gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1187, "groups": "sudo", "home": "/home/gsalomon", "item": {"name": "gsalomon", "ovpn": "gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46"}, "move_home": false, "name": "gsalomon", "shell": "/bin/bash", "state": "present", "uid": 1187} 2021-11-11T20:38:04.246 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'soumyakoduri', 'ovpn': 'skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1188, "groups": "sudo", "home": "/home/soumyakoduri", "item": {"name": "soumyakoduri", "ovpn": "skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133"}, "move_home": false, "name": "soumyakoduri", "shell": "/bin/bash", "state": "present", "uid": 1188} 2021-11-11T20:38:04.299 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'soumyakoduri', 'ovpn': 'skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1188, "groups": "sudo", "home": "/home/soumyakoduri", "item": {"name": "soumyakoduri", "ovpn": "skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133"}, "move_home": false, "name": "soumyakoduri", "shell": "/bin/bash", "state": "present", "uid": 1188} 2021-11-11T20:38:04.445 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kyr', 'ovpn': 'kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1189, "groups": "sudo", "home": "/home/kyr", "item": {"name": "kyr", "ovpn": "kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a"}, "move_home": false, "name": "kyr", "shell": "/bin/bash", "state": "present", "uid": 1189} 2021-11-11T20:38:04.487 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kyr', 'ovpn': 'kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1189, "groups": "sudo", "home": "/home/kyr", "item": {"name": "kyr", "ovpn": "kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a"}, "move_home": false, "name": "kyr", "shell": "/bin/bash", "state": "present", "uid": 1189} 2021-11-11T20:38:04.643 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'sseshasa', 'ovpn': 'sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1190, "groups": "sudo", "home": "/home/sseshasa", "item": {"name": "sseshasa", "ovpn": "sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849"}, "move_home": false, "name": "sseshasa", "shell": "/bin/bash", "state": "present", "uid": 1190} 2021-11-11T20:38:04.685 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'sseshasa', 'ovpn': 'sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1190, "groups": "sudo", "home": "/home/sseshasa", "item": {"name": "sseshasa", "ovpn": "sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849"}, "move_home": false, "name": "sseshasa", "shell": "/bin/bash", "state": "present", "uid": 1190} 2021-11-11T20:38:04.831 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rfriedma', 'ovpn': 'rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1191, "groups": "sudo", "home": "/home/rfriedma", "item": {"name": "rfriedma", "ovpn": "rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189"}, "move_home": false, "name": "rfriedma", "shell": "/bin/bash", "state": "present", "uid": 1191} 2021-11-11T20:38:04.884 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rfriedma', 'ovpn': 'rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1191, "groups": "sudo", "home": "/home/rfriedma", "item": {"name": "rfriedma", "ovpn": "rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189"}, "move_home": false, "name": "rfriedma", "shell": "/bin/bash", "state": "present", "uid": 1191} 2021-11-11T20:38:05.030 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'lmb', 'ovpn': 'lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1192, "groups": "sudo", "home": "/home/lmb", "item": {"name": "lmb", "ovpn": "lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b"}, "move_home": false, "name": "lmb", "shell": "/bin/bash", "state": "present", "uid": 1192} 2021-11-11T20:38:05.073 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'lmb', 'ovpn': 'lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1192, "groups": "sudo", "home": "/home/lmb", "item": {"name": "lmb", "ovpn": "lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b"}, "move_home": false, "name": "lmb", "shell": "/bin/bash", "state": "present", "uid": 1192} 2021-11-11T20:38:05.239 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ksirivad', 'ovpn': 'ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1193, "groups": "sudo", "home": "/home/ksirivad", "item": {"name": "ksirivad", "ovpn": "ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n"}, "move_home": false, "name": "ksirivad", "shell": "/bin/bash", "state": "present", "uid": 1193} 2021-11-11T20:38:05.272 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ksirivad', 'ovpn': 'ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1193, "groups": "sudo", "home": "/home/ksirivad", "item": {"name": "ksirivad", "ovpn": "ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n"}, "move_home": false, "name": "ksirivad", "shell": "/bin/bash", "state": "present", "uid": 1193} 2021-11-11T20:38:05.430 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ofriedma', 'ovpn': 'ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1194, "groups": "sudo", "home": "/home/ofriedma", "item": {"name": "ofriedma", "ovpn": "ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4"}, "move_home": false, "name": "ofriedma", "shell": "/bin/bash", "state": "present", "uid": 1194} 2021-11-11T20:38:05.462 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ofriedma', 'ovpn': 'ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1194, "groups": "sudo", "home": "/home/ofriedma", "item": {"name": "ofriedma", "ovpn": "ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4"}, "move_home": false, "name": "ofriedma", "shell": "/bin/bash", "state": "present", "uid": 1194} 2021-11-11T20:38:05.630 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'h4iresis', 'ovpn': 'h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1195, "groups": "sudo", "home": "/home/h4iresis", "item": {"name": "h4iresis", "ovpn": "h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52"}, "move_home": false, "name": "h4iresis", "shell": "/bin/bash", "state": "present", "uid": 1195} 2021-11-11T20:38:05.651 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'h4iresis', 'ovpn': 'h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1195, "groups": "sudo", "home": "/home/h4iresis", "item": {"name": "h4iresis", "ovpn": "h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52"}, "move_home": false, "name": "h4iresis", "shell": "/bin/bash", "state": "present", "uid": 1195} 2021-11-11T20:38:05.818 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'shyukri', 'ovpn': 'shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1196, "groups": "sudo", "home": "/home/shyukri", "item": {"name": "shyukri", "ovpn": "shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5"}, "move_home": false, "name": "shyukri", "shell": "/bin/bash", "state": "present", "uid": 1196} 2021-11-11T20:38:05.850 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'shyukri', 'ovpn': 'shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1196, "groups": "sudo", "home": "/home/shyukri", "item": {"name": "shyukri", "ovpn": "shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5"}, "move_home": false, "name": "shyukri", "shell": "/bin/bash", "state": "present", "uid": 1196} 2021-11-11T20:38:06.016 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'scarvalhojr', 'ovpn': 'scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1197, "groups": "sudo", "home": "/home/scarvalhojr", "item": {"name": "scarvalhojr", "ovpn": "scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e"}, "move_home": false, "name": "scarvalhojr", "shell": "/bin/bash", "state": "present", "uid": 1197} 2021-11-11T20:38:06.048 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'scarvalhojr', 'ovpn': 'scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1197, "groups": "sudo", "home": "/home/scarvalhojr", "item": {"name": "scarvalhojr", "ovpn": "scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e"}, "move_home": false, "name": "scarvalhojr", "shell": "/bin/bash", "state": "present", "uid": 1197} 2021-11-11T20:38:06.214 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rakesh', 'ovpn': 'rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1198, "groups": "sudo", "home": "/home/rakesh", "item": {"name": "rakesh", "ovpn": "rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b"}, "move_home": false, "name": "rakesh", "shell": "/bin/bash", "state": "present", "uid": 1198} 2021-11-11T20:38:06.246 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rakesh', 'ovpn': 'rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1198, "groups": "sudo", "home": "/home/rakesh", "item": {"name": "rakesh", "ovpn": "rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b"}, "move_home": false, "name": "rakesh", "shell": "/bin/bash", "state": "present", "uid": 1198} 2021-11-11T20:38:06.402 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ryg-', 'ovpn': 'ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1199, "groups": "sudo", "home": "/home/ryg-", "item": {"name": "ryg-", "ovpn": "ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f"}, "move_home": false, "name": "ryg-", "shell": "/bin/bash", "state": "present", "uid": 1199} 2021-11-11T20:38:06.434 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ryg-', 'ovpn': 'ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1199, "groups": "sudo", "home": "/home/ryg-", "item": {"name": "ryg-", "ovpn": "ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f"}, "move_home": false, "name": "ryg-", "shell": "/bin/bash", "state": "present", "uid": 1199} 2021-11-11T20:38:06.600 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1146, "groups": "sudo", "home": "/home/prsrivas", "item": {"name": "prsrivas", "ovpn": "prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3"}, "move_home": false, "name": "prsrivas", "shell": "/bin/bash", "state": "present", "uid": 1146} 2021-11-11T20:38:06.622 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1146, "groups": "sudo", "home": "/home/prsrivas", "item": {"name": "prsrivas", "ovpn": "prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3"}, "move_home": false, "name": "prsrivas", "shell": "/bin/bash", "state": "present", "uid": 1146} 2021-11-11T20:38:06.799 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'zdover', 'ovpn': 'zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1200, "groups": "sudo", "home": "/home/zdover", "item": {"name": "zdover", "ovpn": "zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3"}, "move_home": false, "name": "zdover", "shell": "/bin/bash", "state": "present", "uid": 1200} 2021-11-11T20:38:06.821 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'zdover', 'ovpn': 'zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1200, "groups": "sudo", "home": "/home/zdover", "item": {"name": "zdover", "ovpn": "zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3"}, "move_home": false, "name": "zdover", "shell": "/bin/bash", "state": "present", "uid": 1200} 2021-11-11T20:38:06.987 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rraja', 'ovpn': 'rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1201, "groups": "sudo", "home": "/home/rraja", "item": {"name": "rraja", "ovpn": "rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100"}, "move_home": false, "name": "rraja", "shell": "/bin/bash", "state": "present", "uid": 1201} 2021-11-11T20:38:07.009 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rraja', 'ovpn': 'rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1201, "groups": "sudo", "home": "/home/rraja", "item": {"name": "rraja", "ovpn": "rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100"}, "move_home": false, "name": "rraja", "shell": "/bin/bash", "state": "present", "uid": 1201} 2021-11-11T20:38:07.185 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'toabctl', 'ovpn': 'toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1202, "groups": "sudo", "home": "/home/toabctl", "item": {"name": "toabctl", "ovpn": "toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f"}, "move_home": false, "name": "toabctl", "shell": "/bin/bash", "state": "present", "uid": 1202} 2021-11-11T20:38:07.207 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'toabctl', 'ovpn': 'toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1202, "groups": "sudo", "home": "/home/toabctl", "item": {"name": "toabctl", "ovpn": "toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f"}, "move_home": false, "name": "toabctl", "shell": "/bin/bash", "state": "present", "uid": 1202} 2021-11-11T20:38:07.384 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mgfritch', 'ovpn': 'mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1203, "groups": "sudo", "home": "/home/mgfritch", "item": {"name": "mgfritch", "ovpn": "mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39"}, "move_home": false, "name": "mgfritch", "shell": "/bin/bash", "state": "present", "uid": 1203} 2021-11-11T20:38:07.406 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mgfritch', 'ovpn': 'mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1203, "groups": "sudo", "home": "/home/mgfritch", "item": {"name": "mgfritch", "ovpn": "mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39"}, "move_home": false, "name": "mgfritch", "shell": "/bin/bash", "state": "present", "uid": 1203} 2021-11-11T20:38:07.583 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'xiubli', 'ovpn': 'root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1204, "groups": "sudo", "home": "/home/xiubli", "item": {"name": "xiubli", "ovpn": "root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n"}, "move_home": false, "name": "xiubli", "shell": "/bin/bash", "state": "present", "uid": 1204} 2021-11-11T20:38:07.605 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'xiubli', 'ovpn': 'root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1204, "groups": "sudo", "home": "/home/xiubli", "item": {"name": "xiubli", "ovpn": "root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n"}, "move_home": false, "name": "xiubli", "shell": "/bin/bash", "state": "present", "uid": 1204} 2021-11-11T20:38:07.782 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jafaj', 'ovpn': 'jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1205, "groups": "sudo", "home": "/home/jafaj", "item": {"name": "jafaj", "ovpn": "jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f"}, "move_home": false, "name": "jafaj", "shell": "/bin/bash", "state": "present", "uid": 1205} 2021-11-11T20:38:07.793 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jafaj', 'ovpn': 'jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1205, "groups": "sudo", "home": "/home/jafaj", "item": {"name": "jafaj", "ovpn": "jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f"}, "move_home": false, "name": "jafaj", "shell": "/bin/bash", "state": "present", "uid": 1205} 2021-11-11T20:38:07.980 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jschmid', 'ovpn': 'jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1206, "groups": "sudo", "home": "/home/jschmid", "item": {"name": "jschmid", "ovpn": "jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef"}, "move_home": false, "name": "jschmid", "shell": "/bin/bash", "state": "present", "uid": 1206} 2021-11-11T20:38:07.992 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jschmid', 'ovpn': 'jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1206, "groups": "sudo", "home": "/home/jschmid", "item": {"name": "jschmid", "ovpn": "jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef"}, "move_home": false, "name": "jschmid", "shell": "/bin/bash", "state": "present", "uid": 1206} 2021-11-11T20:38:08.179 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kachau', 'ovpn': 'kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1207, "groups": "sudo", "home": "/home/kachau", "item": {"name": "kachau", "ovpn": "kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe"}, "move_home": false, "name": "kachau", "shell": "/bin/bash", "state": "present", "uid": 1207} 2021-11-11T20:38:08.192 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kachau', 'ovpn': 'kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1207, "groups": "sudo", "home": "/home/kachau", "item": {"name": "kachau", "ovpn": "kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe"}, "move_home": false, "name": "kachau", "shell": "/bin/bash", "state": "present", "uid": 1207} 2021-11-11T20:38:08.369 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'khiremat', 'ovpn': 'khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1208, "groups": "sudo", "home": "/home/khiremat", "item": {"name": "khiremat", "ovpn": "khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315"}, "move_home": false, "name": "khiremat", "shell": "/bin/bash", "state": "present", "uid": 1208} 2021-11-11T20:38:08.381 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'khiremat', 'ovpn': 'khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1208, "groups": "sudo", "home": "/home/khiremat", "item": {"name": "khiremat", "ovpn": "khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315"}, "move_home": false, "name": "khiremat", "shell": "/bin/bash", "state": "present", "uid": 1208} 2021-11-11T20:38:08.578 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kapandya', 'ovpn': 'kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1209, "groups": "sudo", "home": "/home/kapandya", "item": {"name": "kapandya", "ovpn": "kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67"}, "move_home": false, "name": "kapandya", "shell": "/bin/bash", "state": "present", "uid": 1209} 2021-11-11T20:38:08.580 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kapandya', 'ovpn': 'kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1209, "groups": "sudo", "home": "/home/kapandya", "item": {"name": "kapandya", "ovpn": "kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67"}, "move_home": false, "name": "kapandya", "shell": "/bin/bash", "state": "present", "uid": 1209} 2021-11-11T20:38:08.768 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'prasad', 'ovpn': 'prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1210, "groups": "sudo", "home": "/home/prasad", "item": {"name": "prasad", "ovpn": "prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89"}, "move_home": false, "name": "prasad", "shell": "/bin/bash", "state": "present", "uid": 1210} 2021-11-11T20:38:08.770 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'prasad', 'ovpn': 'prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1210, "groups": "sudo", "home": "/home/prasad", "item": {"name": "prasad", "ovpn": "prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89"}, "move_home": false, "name": "prasad", "shell": "/bin/bash", "state": "present", "uid": 1210} 2021-11-11T20:38:08.967 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vikhyat', 'ovpn': 'vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1211, "groups": "sudo", "home": "/home/vikhyat", "item": {"name": "vikhyat", "ovpn": "vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f"}, "move_home": false, "name": "vikhyat", "shell": "/bin/bash", "state": "present", "uid": 1211} 2021-11-11T20:38:08.968 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vikhyat', 'ovpn': 'vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1211, "groups": "sudo", "home": "/home/vikhyat", "item": {"name": "vikhyat", "ovpn": "vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f"}, "move_home": false, "name": "vikhyat", "shell": "/bin/bash", "state": "present", "uid": 1211} 2021-11-11T20:38:09.156 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'shraddhaag', 'ovpn': 'shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1212, "groups": "sudo", "home": "/home/shraddhaag", "item": {"name": "shraddhaag", "ovpn": "shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2"}, "move_home": false, "name": "shraddhaag", "shell": "/bin/bash", "state": "present", "uid": 1212} 2021-11-11T20:38:09.157 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'shraddhaag', 'ovpn': 'shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1212, "groups": "sudo", "home": "/home/shraddhaag", "item": {"name": "shraddhaag", "ovpn": "shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2"}, "move_home": false, "name": "shraddhaag", "shell": "/bin/bash", "state": "present", "uid": 1212} 2021-11-11T20:38:09.355 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'gkyratsas', 'ovpn': 'gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1213, "groups": "sudo", "home": "/home/gkyratsas", "item": {"name": "gkyratsas", "ovpn": "gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7"}, "move_home": false, "name": "gkyratsas", "shell": "/bin/bash", "state": "present", "uid": 1213} 2021-11-11T20:38:09.356 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'gkyratsas', 'ovpn': 'gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1213, "groups": "sudo", "home": "/home/gkyratsas", "item": {"name": "gkyratsas", "ovpn": "gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7"}, "move_home": false, "name": "gkyratsas", "shell": "/bin/bash", "state": "present", "uid": 1213} 2021-11-11T20:38:09.553 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'prateek', 'ovpn': 'prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1214, "groups": "sudo", "home": "/home/prateek", "item": {"name": "prateek", "ovpn": "prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67"}, "move_home": false, "name": "prateek", "shell": "/bin/bash", "state": "present", "uid": 1214} 2021-11-11T20:38:09.555 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'prateek', 'ovpn': 'prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1214, "groups": "sudo", "home": "/home/prateek", "item": {"name": "prateek", "ovpn": "prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67"}, "move_home": false, "name": "prateek", "shell": "/bin/bash", "state": "present", "uid": 1214} 2021-11-11T20:38:09.742 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jelopez', 'ovpn': 'jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1215, "groups": "sudo", "home": "/home/jelopez", "item": {"name": "jelopez", "ovpn": "jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e"}, "move_home": false, "name": "jelopez", "shell": "/bin/bash", "state": "present", "uid": 1215} 2021-11-11T20:38:09.753 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jelopez', 'ovpn': 'jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1215, "groups": "sudo", "home": "/home/jelopez", "item": {"name": "jelopez", "ovpn": "jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e"}, "move_home": false, "name": "jelopez", "shell": "/bin/bash", "state": "present", "uid": 1215} 2021-11-11T20:38:09.942 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dcain', 'ovpn': 'dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1216, "groups": "sudo", "home": "/home/dcain", "item": {"name": "dcain", "ovpn": "dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389"}, "move_home": false, "name": "dcain", "shell": "/bin/bash", "state": "present", "uid": 1216} ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dcain', 'ovpn': 'dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1216, "groups": "sudo", "home": "/home/dcain", "item": {"name": "dcain", "ovpn": "dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389"}, "move_home": false, "name": "dcain", "shell": "/bin/bash", "state": "present", "uid": 1216} 2021-11-11T20:38:10.140 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'sunny', 'ovpn': 'sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1217, "groups": "sudo", "home": "/home/sunny", "item": {"name": "sunny", "ovpn": "sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11"}, "move_home": false, "name": "sunny", "shell": "/bin/bash", "state": "present", "uid": 1217} 2021-11-11T20:38:10.141 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'sunny', 'ovpn': 'sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1217, "groups": "sudo", "home": "/home/sunny", "item": {"name": "sunny", "ovpn": "sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11"}, "move_home": false, "name": "sunny", "shell": "/bin/bash", "state": "present", "uid": 1217} 2021-11-11T20:38:10.328 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'xuxuehan', 'ovpn': 'xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1218, "groups": "sudo", "home": "/home/xuxuehan", "item": {"name": "xuxuehan", "ovpn": "xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n"}, "move_home": false, "name": "xuxuehan", "shell": "/bin/bash", "state": "present", "uid": 1218} 2021-11-11T20:38:10.329 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'xuxuehan', 'ovpn': 'xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1218, "groups": "sudo", "home": "/home/xuxuehan", "item": {"name": "xuxuehan", "ovpn": "xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n"}, "move_home": false, "name": "xuxuehan", "shell": "/bin/bash", "state": "present", "uid": 1218} 2021-11-11T20:38:10.526 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'aclewett', 'ovpn': 'aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1219, "groups": "sudo", "home": "/home/aclewett", "item": {"name": "aclewett", "ovpn": "aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798"}, "move_home": false, "name": "aclewett", "shell": "/bin/bash", "state": "present", "uid": 1219} 2021-11-11T20:38:10.527 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'aclewett', 'ovpn': 'aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1219, "groups": "sudo", "home": "/home/aclewett", "item": {"name": "aclewett", "ovpn": "aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798"}, "move_home": false, "name": "aclewett", "shell": "/bin/bash", "state": "present", "uid": 1219} 2021-11-11T20:38:10.715 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'cblum', 'ovpn': 'cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1220, "groups": "sudo", "home": "/home/cblum", "item": {"name": "cblum", "ovpn": "cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56"}, "move_home": false, "name": "cblum", "shell": "/bin/bash", "state": "present", "uid": 1220} 2021-11-11T20:38:10.716 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'cblum', 'ovpn': 'cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1220, "groups": "sudo", "home": "/home/cblum", "item": {"name": "cblum", "ovpn": "cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56"}, "move_home": false, "name": "cblum", "shell": "/bin/bash", "state": "present", "uid": 1220} 2021-11-11T20:38:10.913 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mlatimer', 'ovpn': 'mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1221, "groups": "sudo", "home": "/home/mlatimer", "item": {"name": "mlatimer", "ovpn": "mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f"}, "move_home": false, "name": "mlatimer", "shell": "/bin/bash", "state": "present", "uid": 1221} 2021-11-11T20:38:10.915 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mlatimer', 'ovpn': 'mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1221, "groups": "sudo", "home": "/home/mlatimer", "item": {"name": "mlatimer", "ovpn": "mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f"}, "move_home": false, "name": "mlatimer", "shell": "/bin/bash", "state": "present", "uid": 1221} 2021-11-11T20:38:11.110 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'shyamsundarr', 'ovpn': 'shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1222, "groups": "sudo", "home": "/home/shyamsundarr", "item": {"name": "shyamsundarr", "ovpn": "shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5"}, "move_home": false, "name": "shyamsundarr", "shell": "/bin/bash", "state": "present", "uid": 1222} 2021-11-11T20:38:11.111 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'shyamsundarr', 'ovpn': 'shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1222, "groups": "sudo", "home": "/home/shyamsundarr", "item": {"name": "shyamsundarr", "ovpn": "shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5"}, "move_home": false, "name": "shyamsundarr", "shell": "/bin/bash", "state": "present", "uid": 1222} 2021-11-11T20:38:11.299 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'shuzhenyi', 'ovpn': 'bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1223, "groups": "sudo", "home": "/home/shuzhenyi", "item": {"name": "shuzhenyi", "ovpn": "bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501"}, "move_home": false, "name": "shuzhenyi", "shell": "/bin/bash", "state": "present", "uid": 1223} 2021-11-11T20:38:11.300 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'shuzhenyi', 'ovpn': 'bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1223, "groups": "sudo", "home": "/home/shuzhenyi", "item": {"name": "shuzhenyi", "ovpn": "bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501"}, "move_home": false, "name": "shuzhenyi", "shell": "/bin/bash", "state": "present", "uid": 1223} 2021-11-11T20:38:11.497 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mdipalma', 'ovpn': 'mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1224, "groups": "sudo", "home": "/home/mdipalma", "item": {"name": "mdipalma", "ovpn": "mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14"}, "move_home": false, "name": "mdipalma", "shell": "/bin/bash", "state": "present", "uid": 1224} 2021-11-11T20:38:11.498 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mdipalma', 'ovpn': 'mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1224, "groups": "sudo", "home": "/home/mdipalma", "item": {"name": "mdipalma", "ovpn": "mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14"}, "move_home": false, "name": "mdipalma", "shell": "/bin/bash", "state": "present", "uid": 1224} 2021-11-11T20:38:11.696 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kchang', 'ovpn': 'kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1225, "groups": "sudo", "home": "/home/kchang", "item": {"name": "kchang", "ovpn": "kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3"}, "move_home": false, "name": "kchang", "shell": "/bin/bash", "state": "present", "uid": 1225} 2021-11-11T20:38:11.698 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kchang', 'ovpn': 'kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1225, "groups": "sudo", "home": "/home/kchang", "item": {"name": "kchang", "ovpn": "kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3"}, "move_home": false, "name": "kchang", "shell": "/bin/bash", "state": "present", "uid": 1225} 2021-11-11T20:38:11.884 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vereddy', 'ovpn': 'vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1226, "groups": "sudo", "home": "/home/vereddy", "item": {"name": "vereddy", "ovpn": "vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2"}, "move_home": false, "name": "vereddy", "shell": "/bin/bash", "state": "present", "uid": 1226} 2021-11-11T20:38:11.885 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vereddy', 'ovpn': 'vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1226, "groups": "sudo", "home": "/home/vereddy", "item": {"name": "vereddy", "ovpn": "vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2"}, "move_home": false, "name": "vereddy", "shell": "/bin/bash", "state": "present", "uid": 1226} 2021-11-11T20:38:12.083 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'benhanokh', 'ovpn': 'gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1227, "groups": "sudo", "home": "/home/benhanokh", "item": {"name": "benhanokh", "ovpn": "gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5"}, "move_home": false, "name": "benhanokh", "shell": "/bin/bash", "state": "present", "uid": 1227} 2021-11-11T20:38:12.085 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'benhanokh', 'ovpn': 'gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1227, "groups": "sudo", "home": "/home/benhanokh", "item": {"name": "benhanokh", "ovpn": "gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5"}, "move_home": false, "name": "benhanokh", "shell": "/bin/bash", "state": "present", "uid": 1227} 2021-11-11T20:38:12.272 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jsalomon', 'ovpn': 'jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1228, "groups": "sudo", "home": "/home/jsalomon", "item": {"name": "jsalomon", "ovpn": "jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0"}, "move_home": false, "name": "jsalomon", "shell": "/bin/bash", "state": "present", "uid": 1228} 2021-11-11T20:38:12.283 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jsalomon', 'ovpn': 'jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1228, "groups": "sudo", "home": "/home/jsalomon", "item": {"name": "jsalomon", "ovpn": "jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0"}, "move_home": false, "name": "jsalomon", "shell": "/bin/bash", "state": "present", "uid": 1228} 2021-11-11T20:38:12.470 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jolmomar', 'ovpn': 'jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1229, "groups": "sudo", "home": "/home/jolmomar", "item": {"name": "jolmomar", "ovpn": "jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640"}, "move_home": false, "name": "jolmomar", "shell": "/bin/bash", "state": "present", "uid": 1229} 2021-11-11T20:38:12.472 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jolmomar', 'ovpn': 'jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1229, "groups": "sudo", "home": "/home/jolmomar", "item": {"name": "jolmomar", "ovpn": "jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640"}, "move_home": false, "name": "jolmomar", "shell": "/bin/bash", "state": "present", "uid": 1229} 2021-11-11T20:38:12.668 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'julpark', 'ovpn': 'julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1231, "groups": "sudo", "home": "/home/julpark", "item": {"name": "julpark", "ovpn": "julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8"}, "move_home": false, "name": "julpark", "shell": "/bin/bash", "state": "present", "uid": 1231} 2021-11-11T20:38:12.669 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'julpark', 'ovpn': 'julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1231, "groups": "sudo", "home": "/home/julpark", "item": {"name": "julpark", "ovpn": "julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8"}, "move_home": false, "name": "julpark", "shell": "/bin/bash", "state": "present", "uid": 1231} 2021-11-11T20:38:12.866 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'pdhange', 'ovpn': 'pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1232, "groups": "sudo", "home": "/home/pdhange", "item": {"name": "pdhange", "ovpn": "pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535"}, "move_home": false, "name": "pdhange", "shell": "/bin/bash", "state": "present", "uid": 1232} 2021-11-11T20:38:12.867 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'pdhange', 'ovpn': 'pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1232, "groups": "sudo", "home": "/home/pdhange", "item": {"name": "pdhange", "ovpn": "pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535"}, "move_home": false, "name": "pdhange", "shell": "/bin/bash", "state": "present", "uid": 1232} 2021-11-11T20:38:13.069 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dpivonka', 'ovpn': 'dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1233, "groups": "sudo", "home": "/home/dpivonka", "item": {"name": "dpivonka", "ovpn": "dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4"}, "move_home": false, "name": "dpivonka", "shell": "/bin/bash", "state": "present", "uid": 1233} 2021-11-11T20:38:13.070 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dpivonka', 'ovpn': 'dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1233, "groups": "sudo", "home": "/home/dpivonka", "item": {"name": "dpivonka", "ovpn": "dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4"}, "move_home": false, "name": "dpivonka", "shell": "/bin/bash", "state": "present", "uid": 1233} 2021-11-11T20:38:13.257 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'adking', 'ovpn': 'adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1234, "groups": "sudo", "home": "/home/adking", "item": {"name": "adking", "ovpn": "adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625"}, "move_home": false, "name": "adking", "shell": "/bin/bash", "state": "present", "uid": 1234} 2021-11-11T20:38:13.258 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'adking', 'ovpn': 'adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1234, "groups": "sudo", "home": "/home/adking", "item": {"name": "adking", "ovpn": "adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625"}, "move_home": false, "name": "adking", "shell": "/bin/bash", "state": "present", "uid": 1234} 2021-11-11T20:38:13.456 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'amathuri', 'ovpn': 'amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1235, "groups": "sudo", "home": "/home/amathuri", "item": {"name": "amathuri", "ovpn": "amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n"}, "move_home": false, "name": "amathuri", "shell": "/bin/bash", "state": "present", "uid": 1235} 2021-11-11T20:38:13.457 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'amathuri', 'ovpn': 'amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1235, "groups": "sudo", "home": "/home/amathuri", "item": {"name": "amathuri", "ovpn": "amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n"}, "move_home": false, "name": "amathuri", "shell": "/bin/bash", "state": "present", "uid": 1235} 2021-11-11T20:38:13.645 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'nizamial09', 'ovpn': 'nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1236, "groups": "sudo", "home": "/home/nizamial09", "item": {"name": "nizamial09", "ovpn": "nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409"}, "move_home": false, "name": "nizamial09", "shell": "/bin/bash", "state": "present", "uid": 1236} 2021-11-11T20:38:13.646 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'nizamial09', 'ovpn': 'nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1236, "groups": "sudo", "home": "/home/nizamial09", "item": {"name": "nizamial09", "ovpn": "nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409"}, "move_home": false, "name": "nizamial09", "shell": "/bin/bash", "state": "present", "uid": 1236} 2021-11-11T20:38:13.844 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dachary', 'ovpn': 'loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1237, "groups": "sudo", "home": "/home/dachary", "item": {"name": "dachary", "ovpn": "loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5"}, "move_home": false, "name": "dachary", "shell": "/bin/bash", "state": "present", "uid": 1237} 2021-11-11T20:38:13.844 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dachary', 'ovpn': 'loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5'}) => {"ansible_loop_var": "item", "append": true, "changed": false, "comment": "", "group": 1237, "groups": "sudo", "home": "/home/dachary", "item": {"name": "dachary", "ovpn": "loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5"}, "move_home": false, "name": "dachary", "shell": "/bin/bash", "state": "present", "uid": 1237} 2021-11-11T20:38:15.420 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'amctagga', 'ovpn': 'amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10102, "groups": "sudo", "home": "/home/amctagga", "item": {"name": "amctagga", "ovpn": "amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1"}, "name": "amctagga", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10102} 2021-11-11T20:38:15.421 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'amctagga', 'ovpn': 'amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10102, "groups": "sudo", "home": "/home/amctagga", "item": {"name": "amctagga", "ovpn": "amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1"}, "name": "amctagga", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10102} 2021-11-11T20:38:17.352 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'kevinz', 'ovpn': 'kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10103, "groups": "sudo", "home": "/home/kevinz", "item": {"name": "kevinz", "ovpn": "kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f"}, "name": "kevinz", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10103} 2021-11-11T20:38:17.353 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'kevinz', 'ovpn': 'kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10103, "groups": "sudo", "home": "/home/kevinz", "item": {"name": "kevinz", "ovpn": "kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f"}, "name": "kevinz", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10103} 2021-11-11T20:38:18.843 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'liu-chunmei', 'ovpn': 'liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10104, "groups": "sudo", "home": "/home/liu-chunmei", "item": {"name": "liu-chunmei", "ovpn": "liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c"}, "name": "liu-chunmei", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10104} 2021-11-11T20:38:18.886 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'liu-chunmei', 'ovpn': 'liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10104, "groups": "sudo", "home": "/home/liu-chunmei", "item": {"name": "liu-chunmei", "ovpn": "liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c"}, "name": "liu-chunmei", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10104} 2021-11-11T20:38:20.355 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'cyx1231st', 'ovpn': 'cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10105, "groups": "sudo", "home": "/home/cyx1231st", "item": {"name": "cyx1231st", "ovpn": "cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a"}, "name": "cyx1231st", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10105} 2021-11-11T20:38:20.428 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'cyx1231st', 'ovpn': 'cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10105, "groups": "sudo", "home": "/home/cyx1231st", "item": {"name": "cyx1231st", "ovpn": "cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a"}, "name": "cyx1231st", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10105} 2021-11-11T20:38:21.833 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'sunnagar', 'ovpn': 'sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10106, "groups": "sudo", "home": "/home/sunnagar", "item": {"name": "sunnagar", "ovpn": "sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae"}, "name": "sunnagar", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10106} 2021-11-11T20:38:22.030 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'sunnagar', 'ovpn': 'sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10106, "groups": "sudo", "home": "/home/sunnagar", "item": {"name": "sunnagar", "ovpn": "sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae"}, "name": "sunnagar", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10106} 2021-11-11T20:38:23.387 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'lflores', 'ovpn': 'lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10107, "groups": "sudo", "home": "/home/lflores", "item": {"name": "lflores", "ovpn": "lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2"}, "name": "lflores", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10107} 2021-11-11T20:38:23.543 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'lflores', 'ovpn': 'lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10107, "groups": "sudo", "home": "/home/lflores", "item": {"name": "lflores", "ovpn": "lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2"}, "name": "lflores", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10107} 2021-11-11T20:38:24.883 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'jsawaya', 'ovpn': 'jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10108, "groups": "sudo", "home": "/home/jsawaya", "item": {"name": "jsawaya", "ovpn": "jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46"}, "name": "jsawaya", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10108} 2021-11-11T20:38:25.092 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'jsawaya', 'ovpn': 'jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10108, "groups": "sudo", "home": "/home/jsawaya", "item": {"name": "jsawaya", "ovpn": "jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46"}, "name": "jsawaya", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10108} 2021-11-11T20:38:26.374 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'markw8973240', 'ovpn': 'shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10109, "groups": "sudo", "home": "/home/markw8973240", "item": {"name": "markw8973240", "ovpn": "shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d"}, "name": "markw8973240", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10109} 2021-11-11T20:38:26.653 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'markw8973240', 'ovpn': 'shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10109, "groups": "sudo", "home": "/home/markw8973240", "item": {"name": "markw8973240", "ovpn": "shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d"}, "name": "markw8973240", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10109} 2021-11-11T20:38:27.933 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'luisrico', 'ovpn': 'lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10110, "groups": "sudo", "home": "/home/luisrico", "item": {"name": "luisrico", "ovpn": "lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347"}, "name": "luisrico", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10110} 2021-11-11T20:38:28.255 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'luisrico', 'ovpn': 'lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10110, "groups": "sudo", "home": "/home/luisrico", "item": {"name": "luisrico", "ovpn": "lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347"}, "name": "luisrico", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10110} 2021-11-11T20:38:29.434 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'matan', 'ovpn': 'matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10111, "groups": "sudo", "home": "/home/matan", "item": {"name": "matan", "ovpn": "matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n"}, "name": "matan", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10111} 2021-11-11T20:38:29.786 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'matan', 'ovpn': 'matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10111, "groups": "sudo", "home": "/home/matan", "item": {"name": "matan", "ovpn": "matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n"}, "name": "matan", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10111} 2021-11-11T20:38:30.983 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'ozeneva', 'ovpn': 'ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10112, "groups": "sudo", "home": "/home/ozeneva", "item": {"name": "ozeneva", "ovpn": "ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5"}, "name": "ozeneva", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10112} 2021-11-11T20:38:31.355 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'ozeneva', 'ovpn': 'ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10112, "groups": "sudo", "home": "/home/ozeneva", "item": {"name": "ozeneva", "ovpn": "ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5"}, "name": "ozeneva", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10112} 2021-11-11T20:38:32.595 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'sudhanaruna', 'ovpn': 'ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10113, "groups": "sudo", "home": "/home/sudhanaruna", "item": {"name": "sudhanaruna", "ovpn": "ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997"}, "name": "sudhanaruna", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10113} 2021-11-11T20:38:32.916 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'sudhanaruna', 'ovpn': 'ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10113, "groups": "sudo", "home": "/home/sudhanaruna", "item": {"name": "sudhanaruna", "ovpn": "ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997"}, "name": "sudhanaruna", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10113} 2021-11-11T20:38:34.219 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'zl31wang', 'ovpn': 'zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10114, "groups": "sudo", "home": "/home/zl31wang", "item": {"name": "zl31wang", "ovpn": "zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2"}, "name": "zl31wang", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10114} 2021-11-11T20:38:34.551 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'zl31wang', 'ovpn': 'zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10114, "groups": "sudo", "home": "/home/zl31wang", "item": {"name": "zl31wang", "ovpn": "zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2"}, "name": "zl31wang", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10114} 2021-11-11T20:38:35.730 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'mayagilad', 'ovpn': 'mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10115, "groups": "sudo", "home": "/home/mayagilad", "item": {"name": "mayagilad", "ovpn": "mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0"}, "name": "mayagilad", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10115} 2021-11-11T20:38:36.174 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'mayagilad', 'ovpn': 'mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10115, "groups": "sudo", "home": "/home/mayagilad", "item": {"name": "mayagilad", "ovpn": "mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0"}, "name": "mayagilad", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10115} 2021-11-11T20:38:37.465 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'arjsharm', 'ovpn': 'arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10116, "groups": "sudo", "home": "/home/arjsharm", "item": {"name": "arjsharm", "ovpn": "arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39"}, "name": "arjsharm", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10116} 2021-11-11T20:38:37.745 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'arjsharm', 'ovpn': 'arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10116, "groups": "sudo", "home": "/home/arjsharm", "item": {"name": "arjsharm", "ovpn": "arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39"}, "name": "arjsharm", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10116} 2021-11-11T20:38:39.116 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'melissali', 'ovpn': 'melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10117, "groups": "sudo", "home": "/home/melissali", "item": {"name": "melissali", "ovpn": "melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07"}, "name": "melissali", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10117} 2021-11-11T20:38:39.436 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'melissali', 'ovpn': 'melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10117, "groups": "sudo", "home": "/home/melissali", "item": {"name": "melissali", "ovpn": "melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07"}, "name": "melissali", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10117} 2021-11-11T20:38:40.740 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'ionutbalutoiu', 'ovpn': 'ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10118, "groups": "sudo", "home": "/home/ionutbalutoiu", "item": {"name": "ionutbalutoiu", "ovpn": "ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1"}, "name": "ionutbalutoiu", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10118} 2021-11-11T20:38:41.060 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'ionutbalutoiu', 'ovpn': 'ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10118, "groups": "sudo", "home": "/home/ionutbalutoiu", "item": {"name": "ionutbalutoiu", "ovpn": "ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1"}, "name": "ionutbalutoiu", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10118} 2021-11-11T20:38:42.400 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'pdiazbou', 'ovpn': 'pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10119, "groups": "sudo", "home": "/home/pdiazbou", "item": {"name": "pdiazbou", "ovpn": "pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4"}, "name": "pdiazbou", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10119} 2021-11-11T20:38:42.730 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'pdiazbou', 'ovpn': 'pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10119, "groups": "sudo", "home": "/home/pdiazbou", "item": {"name": "pdiazbou", "ovpn": "pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4"}, "name": "pdiazbou", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10119} 2021-11-11T20:38:44.034 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'xuwang', 'ovpn': 'xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10120, "groups": "sudo", "home": "/home/xuwang", "item": {"name": "xuwang", "ovpn": "xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8"}, "name": "xuwang", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10120} 2021-11-11T20:38:44.379 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'xuwang', 'ovpn': 'xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10120, "groups": "sudo", "home": "/home/xuwang", "item": {"name": "xuwang", "ovpn": "xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8"}, "name": "xuwang", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10120} 2021-11-11T20:38:45.691 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'blaine', 'ovpn': 'blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10121, "groups": "sudo", "home": "/home/blaine", "item": {"name": "blaine", "ovpn": "blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547"}, "name": "blaine", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10121} 2021-11-11T20:38:45.951 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'blaine', 'ovpn': 'blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10121, "groups": "sudo", "home": "/home/blaine", "item": {"name": "blaine", "ovpn": "blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547"}, "name": "blaine", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10121} 2021-11-11T20:38:47.378 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'choffman', 'ovpn': 'choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10122, "groups": "sudo", "home": "/home/choffman", "item": {"name": "choffman", "ovpn": "choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c"}, "name": "choffman", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10122} 2021-11-11T20:38:47.509 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:38:47 +0000 (0:00:59.632) 0:01:07.765 ***** 2021-11-11T20:38:47.579 INFO:teuthology.task.ansible.out: TASK [Create all users without sudo access.] *********************************** 2021-11-11T20:38:47.601 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:38:47 +0000 (0:00:00.092) 0:01:07.857 ***** 2021-11-11T20:38:47.651 INFO:teuthology.task.ansible.out: TASK [Create all admin users with sudo access.] ******************************** 2021-11-11T20:38:47.652 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'choffman', 'ovpn': 'choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c'}) => {"ansible_loop_var": "item", "changed": true, "comment": "", "create_home": true, "group": 10122, "groups": "sudo", "home": "/home/choffman", "item": {"name": "choffman", "ovpn": "choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c"}, "name": "choffman", "shell": "/bin/bash", "state": "present", "system": false, "uid": 10122} 2021-11-11T20:38:47.786 INFO:teuthology.task.ansible.out: TASK [Merge managed_users and managed_admin users] ***************************** 2021-11-11T20:38:47.788 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"pubkey_users": [{"name": "andrewschoen", "ovpn": "andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9"}, {"name": "zack", "ovpn": "zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n"}, {"name": "kdreyer", "ovpn": "kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd"}, {"name": "dmick", "ovpn": "dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n"}, {"name": "sage", "ovpn": "sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n"}, {"name": "dgalloway", "ovpn": "dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n"}, {"name": "akraitma", "ovpn": "akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045"}, {"name": "jcaratzas", "ovpn": "jcaratza@desktop 2021-11-11T20:38:47.789 INFO:teuthology.task.ansible.out: QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898"}, {"name": "trociny", "ovpn": "mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e"}, {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf", "name": "smithfarm", "ovpn": "smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007"}, {"name": "mbenjamin", "ovpn": "mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n"}, {"name": "aemerson", "ovpn": "aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n"}, {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour", "name": "amaredia", "ovpn": "ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b26 2021-11-11T20:38:47.791 INFO:teuthology.task.ansible.out:5c63f3ea91e7dacd78"}, {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet", "name": "tserlin", "ovpn": "tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5"}, {"name": "dis", "ovpn": "dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28"}, {"name": "gregf", "ovpn": "gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n"}, {"name": "joshd", "ovpn": "jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n"}, {"name": "cmeno", "ovpn": "cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed"}, {"name": "ivancich", "ovpn": "ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1"}, {"name": "yuriw", "ovpn": "yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed20 2021-11-11T20:38:47.793 INFO:teuthology.task.ansible.out:82caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n"}, {"name": "tamil", "ovpn": "tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n"}, {"name": "jowilkin", "ovpn": "jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n"}, {"name": "bhubbard", "ovpn": "brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4"}, {"name": "yehudasa", "ovpn": "yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n"}, {"name": "dang", "ovpn": "dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4"}, {"name": "branto", "ovpn": "branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb"}, {"name": "xiaoxichen", "ovpn": "xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6"}, {"name": "ffilz", "ovpn": "ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e"}, {"name" 2021-11-11T20:38:47.795 INFO:teuthology.task.ansible.out:: "joao", "ovpn": "joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n"}, {"name": "nhm", "ovpn": "nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n"}, {"name": "mkidd", "ovpn": "linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22"}, {"name": "jlopez", "ovpn": "JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af"}, {"name": "haomaiwang", "ovpn": "yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb"}, {"name": "kchai", "ovpn": "kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3"}, {"name": "vumrao", "ovpn": "vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2"}, {"name": "dfuller", "ovpn": "dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n"}, {"name": "owasserm", "ovpn": "owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n"}, {"name": "vasu" 2021-11-11T20:38:47.795 INFO:teuthology.task.ansible.out:, "ovpn": "vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915 2021-11-11T20:38:47.797 INFO:teuthology.task.ansible.out:d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n"}, {"name": "smohan", "ovpn": "shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37"}, {"name": "abhi", "ovpn": "abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114"}, {"name": "cbodley", "ovpn": "cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611"}, {"name": "mwatts", "ovpn": "mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195"}, {"name": "prsrivas", "ovpn": "pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n"}, {"name": "pdonnell", "ovpn": "pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea"}, {"name": "jlayton", "ovpn": "jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868"}, {"name": "rzarzynski", "ovpn": "rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f"}, {"name": "rdias", "ovpn": "rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e"}, {"name": "vshankar", "ovpn": "vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20"}, {"name": "akupczyk", "ovpn": "adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58"}, {"name": "nojha", "ovpn": "nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n"}, {"name": "ifed01", "ovpn": "ifed01@snowmon CjHOevVvBm+PwhjnhZ 2021-11-11T20:38:47.798 INFO:teuthology.task.ansible.out:YM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56"}, {"name": "myoungwon", "ovpn": "myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71"}, {"name": "gabrioux", "ovpn": "gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2"}, {"name": "leseb", "ovpn": "leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f"}, {"name": "hchen", "ovpn": "hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca"}, {"name": "jcollin", "ovpn": "jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n"}, {"name": "xxg", "ovpn": "xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc"}, {"name": "pcuzner", "ovpn": "pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53"}, {"name": "mkogan", "ovpn": "mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2"}, {"name": "vumrao", "ovpn": "vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed"}, {"name": "henrix", "ovpn": "henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736"}, {"name": "pbs1108", "ovpn": "pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26"}, {"name": "clacroix", "ovpn": "clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315"}, {"name": "epuertat", "ovpn": "epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006"}, {"name": "tdehler", "ovpn": "tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 2021-11-11T20:38:47.800 INFO:teuthology.task.ansible.out: 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461"}, {"name": "laura", "ovpn": "laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305"}, {"name": "yaarit", "ovpn": "yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce"}, {"name": "rpavani1998", "ovpn": "admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3"}, {"name": "rishabh", "ovpn": "rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4"}, {"name": "skrah", "ovpn": "skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2"}, {"name": "smanjara", "ovpn": "smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f"}, {"name": "bengland", "ovpn": "bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41"}, {"name": "pnawracay", "ovpn": "pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d"}, {"name": "alfonsomthd", "ovpn": "alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47"}, {"name": "oliveiradan", "ovpn": "oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e"}, {"name": "swagner", "ovpn": "swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba"}, {"name": "yuvalif", "ovpn": "ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n"}, {"name": "kkeithle", "ovpn": "kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852"}, {"name": "emmericp 2021-11-11T20:38:47.802 INFO:teuthology.task.ansible.out:", "ovpn": "paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130"}, {"name": "mchangir", "ovpn": "mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0"}, {"name": "sjust", "ovpn": "sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n"}, {"name": "ideepika", "ovpn": "deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f"}, {"name": "gsalomon", "ovpn": "gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46"}, {"name": "soumyakoduri", "ovpn": "skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133"}, {"name": "kyr", "ovpn": "kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a"}, {"name": "sseshasa", "ovpn": "sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849"}, {"name": "rfriedma", "ovpn": "rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189"}, {"name": "lmb", "ovpn": "lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b"}, {"name": "ksirivad", "ovpn": "ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n"}, {"name": "ofriedma", "ovpn": "ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3 2021-11-11T20:38:47.803 INFO:teuthology.task.ansible.out:624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4"}, {"name": "h4iresis", "ovpn": "h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52"}, {"name": "shyukri", "ovpn": "shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5"}, {"name": "scarvalhojr", "ovpn": "scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e"}, {"name": "rakesh", "ovpn": "rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b"}, {"name": "ryg-", "ovpn": "ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f"}, {"name": "prsrivas", "ovpn": "prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3"}, {"name": "zdover", "ovpn": "zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3"}, {"name": "rraja", "ovpn": "rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100"}, {"name": "toabctl", "ovpn": "toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f"}, {"name": "mgfritch", "ovpn": "mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39"}, {"name": "xiubli", "ovpn": "root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n"}, {"name": "jafaj", "ovpn": "jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f"}, {"name": "jschmid", "ovpn": "jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef"}, {"name": "kachau", "ovpn": "kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe"}, {"name": "khiremat", "ovpn": "k 2021-11-11T20:38:47.804 INFO:teuthology.task.ansible.out:hiremat@kotresh-T490s 7o9Jj 2021-11-11T20:38:47.805 INFO:teuthology.task.ansible.out:enDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315"}, {"name": "kapandya", "ovpn": "kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67"}, {"name": "prasad", "ovpn": "prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89"}, {"name": "vikhyat", "ovpn": "vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f"}, {"name": "shraddhaag", "ovpn": "shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2"}, {"name": "gkyratsas", "ovpn": "gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7"}, {"name": "prateek", "ovpn": "prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67"}, {"name": "jelopez", "ovpn": "jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e"}, {"name": "dcain", "ovpn": "dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389"}, {"name": "sunny", "ovpn": "sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11"}, {"name": "xuxuehan", "ovpn": "xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n"}, {"name": "aclewett", "ovpn": "aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798"}, {"name": "cblum", "ovpn": "cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56"}, {"name": "mlatimer", "ovpn": "mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f"}, {"name": "shyamsundarr", "ovpn": "sh 2021-11-11T20:38:47.807 INFO:teuthology.task.ansible.out:yamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5"}, {"name": "shuzhenyi", "ovpn": "bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501"}, {"name": "mdipalma", "ovpn": "mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14"}, {"name": "kchang", "ovpn": "kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3"}, {"name": "vereddy", "ovpn": "vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2"}, {"name": "benhanokh", "ovpn": "gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5"}, {"name": "jsalomon", "ovpn": "jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0"}, {"name": "jolmomar", "ovpn": "jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640"}, {"name": "julpark", "ovpn": "julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8"}, {"name": "pdhange", "ovpn": "pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535"}, {"name": "dpivonka", "ovpn": "dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4"}, {"name": "adking", "ovpn": "adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625"}, {"name": "amathuri", "ovpn": "amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n"}, {"name": "nizamial09", "ovpn": "nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a 2021-11-11T20:38:47.808 INFO:teuthology.task.ansible.out:28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409"}, {"name": "dachary", "ovpn": "loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5"}, {"name": "amctagga", "ovpn": "amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1"}, {"name": "kevinz", "ovpn": "kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f"}, {"name": "liu-chunmei", "ovpn": "liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c"}, {"name": "cyx1231st", "ovpn": "cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a"}, {"name": "sunnagar", "ovpn": "sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae"}, {"name": "lflores", "ovpn": "lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2"}, {"name": "jsawaya", "ovpn": "jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46"}, {"name": "markw8973240", "ovpn": "shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d"}, {"name": "luisrico", "ovpn": "lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347"}, {"name": "matan", "ovpn": "matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n"}, {"name": "ozeneva", "ovpn": "ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5"}, {"name": "sudhanaruna", "ovpn": "ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997"}, {"name": "zl31wang", "ovpn": " 2021-11-11T20:38:47.810 INFO:teuthology.task.ansible.out:zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2"}, {"name": "mayagilad", "ovpn": "mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0"}, {"name": "arjsharm", "ovpn": "arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39"}, {"name": "melissali", "ovpn": "melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07"}, {"name": "ionutbalutoiu", "ovpn": "ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1"}, {"name": "pdiazbou", "ovpn": "pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4"}, {"name": "xuwang", "ovpn": "xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8"}, {"name": "blaine", "ovpn": "blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547"}, {"name": "choffman", "ovpn": "choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c"}]}, "changed": false} 2021-11-11T20:38:47.827 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:38:47 +0000 (0:00:00.225) 0:01:08.083 ***** 2021-11-11T20:38:47.850 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:38:47 +0000 (0:00:00.022) 0:01:08.105 ***** 2021-11-11T20:38:47.892 INFO:teuthology.task.ansible.out: TASK [Create all users without sudo access.] *********************************** 2021-11-11T20:38:47.914 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:38:47 +0000 (0:00:00.064) 0:01:08.170 ***** 2021-11-11T20:38:48.007 INFO:teuthology.task.ansible.out: TASK [Merge managed_users and managed_admin users] ***************************** 2021-11-11T20:38:48.009 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"pubkey_users": [{"name": "andrewschoen", "ovpn": "andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9"}, {"name": "zack", "ovpn": "zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n"}, {"name": "kdreyer", "ovpn": "kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd"}, {"name": "dmick", "ovpn": "dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n"}, {"name": "sage", "ovpn": "sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n"}, {"name": "dgalloway", "ovpn": "dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n"}, {"name": "akraitma", "ovpn": "akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045"}, {"name": "jcaratzas", "ovpn": "jcaratza@desktop 2021-11-11T20:38:48.010 INFO:teuthology.task.ansible.out: QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898"}, {"name": "trociny", "ovpn": "mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e"}, {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf", "name": "smithfarm", "ovpn": "smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007"}, {"name": "mbenjamin", "ovpn": "mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n"}, {"name": "aemerson", "ovpn": "aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n"}, {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour", "name": "amaredia", "ovpn": "ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b26 2021-11-11T20:38:48.012 INFO:teuthology.task.ansible.out:5c63f3ea91e7dacd78"}, {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet", "name": "tserlin", "ovpn": "tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5"}, {"name": "dis", "ovpn": "dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28"}, {"name": "gregf", "ovpn": "gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n"}, {"name": "joshd", "ovpn": "jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n"}, {"name": "cmeno", "ovpn": "cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed"}, {"name": "ivancich", "ovpn": "ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1"}, {"name": "yuriw", "ovpn": "yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed20 2021-11-11T20:38:48.014 INFO:teuthology.task.ansible.out:82caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n"}, {"name": "tamil", "ovpn": "tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n"}, {"name": "jowilkin", "ovpn": "jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n"}, {"name": "bhubbard", "ovpn": "brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4"}, {"name": "yehudasa", "ovpn": "yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n"}, {"name": "dang", "ovpn": "dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4"}, {"name": "branto", "ovpn": "branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb"}, {"name": "xiaoxichen", "ovpn": "xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6"}, {"name": "ffilz", "ovpn": "ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e"}, {"name" 2021-11-11T20:38:48.015 INFO:teuthology.task.ansible.out:: "joao", "ovpn": "joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9eb 2021-11-11T20:38:48.016 INFO:teuthology.task.ansible.out:ba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n"}, {"name": "nhm", "ovpn": "nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n"}, {"name": "mkidd", "ovpn": "linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22"}, {"name": "jlopez", "ovpn": "JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af"}, {"name": "haomaiwang", "ovpn": "yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb"}, {"name": "kchai", "ovpn": "kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3"}, {"name": "vumrao", "ovpn": "vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2"}, {"name": "dfuller", "ovpn": "dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n"}, {"name": "owasserm", "ovpn": "owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n"}, {"name": "vasu", "ovpn": "vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915 2021-11-11T20:38:48.018 INFO:teuthology.task.ansible.out:d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n"}, {"name": "smohan", "ovpn": "shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37"}, {"name": "abhi", "ovpn": "abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114"}, {"name": "cbodley", "ovpn": "cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611"}, {"name": "mwatts", "ovpn": "mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195"}, {"name": "prsrivas", "ovpn": "pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n"}, {"name": "pdonnell", "ovpn": "pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea"}, {"name": "jlayton", "ovpn": "jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868"}, {"name": "rzarzynski", "ovpn": "rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f"}, {"name": "rdias", "ovpn": "rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e"}, {"name": "vshankar", "ovpn": "vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20"}, {"name": "akupczyk", "ovpn": "adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58"}, {"name": "nojha", "ovpn": "nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n"}, {"name": "ifed01", "ovpn": "ifed01@snowmon CjHOevVvBm+PwhjnhZ 2021-11-11T20:38:48.020 INFO:teuthology.task.ansible.out:YM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56"}, {"name": "myoungwon", "ovpn": "myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71"}, {"name": "gabrioux", "ovpn": "gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2"}, {"name": "leseb", "ovpn": "leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f"}, {"name": "hchen", "ovpn": "hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca"}, {"name": "jcollin", "ovpn": "jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n"}, {"name": "xxg", "ovpn": "xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc"}, {"name": "pcuzner", "ovpn": "pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53"}, {"name": "mkogan", "ovpn": "mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2"}, {"name": "vumrao", "ovpn": "vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed"}, {"name": "henrix", "ovpn": "henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736"}, {"name": "pbs1108", "ovpn": "pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26"}, {"name": "clacroix", "ovpn": "clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315"}, {"name": "epuertat", "ovpn": "epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006"}, {"name": "tdehler", "ovpn": "tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 2021-11-11T20:38:48.022 INFO:teuthology.task.ansible.out: 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461"}, {"name": "laura", "ovpn": "laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305"}, {"name": "yaarit", "ovpn": "yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce"}, {"name": "rpavani1998", "ovpn": "admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3"}, {"name": "rishabh", "ovpn": "rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4"}, {"name": "skrah", "ovpn": "skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2"}, {"name": "smanjara", "ovpn": "smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f"}, {"name": "bengland", "ovpn": "bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41"}, {"name": "pnawracay", "ovpn": "pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d"}, {"name": "alfonsomthd", "ovpn": "alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47"}, {"name": "oliveiradan", "ovpn": "oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e"}, {"name": "swagner", "ovpn": "swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba"}, {"name": "yuvalif", "ovpn": "ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n"}, {"name": "kkeithle", "ovpn": "kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852"}, {"name": "emmericp 2021-11-11T20:38:48.023 INFO:teuthology.task.ansible.out:", "ovpn": "paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130"}, {"name": "mchangir", "ovpn": "mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0"}, {"name": "sjust", "ovpn": "sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n"}, {"name": "ideepika", "ovpn": "deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f"}, {"name": "gsalomon", "ovpn": "gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46"}, {"name": "soumyakoduri", "ovpn": "skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133"}, {"name": "kyr", "ovpn": "kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a"}, {"name": "sseshasa", "ovpn": "sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849"}, {"name": "rfriedma", "ovpn": "rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189"}, {"name": "lmb", "ovpn": "lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b"}, {"name": "ksirivad", "ovpn": "ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n"}, {"name": "ofriedma", "ovpn": "ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3 2021-11-11T20:38:48.025 INFO:teuthology.task.ansible.out:624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4"}, {"name": "h4iresis", "ovpn": "h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52"}, {"name": "shyukri", "ovpn": "shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5"}, {"name": "scarvalhojr", "ovpn": "scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e"}, {"name": "rakesh", "ovpn": "rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b"}, {"name": "ryg-", "ovpn": "ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f"}, {"name": "prsrivas", "ovpn": "prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3"}, {"name": "zdover", "ovpn": "zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3"}, {"name": "rraja", "ovpn": "rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100"}, {"name": "toabctl", "ovpn": "toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f"}, {"name": "mgfritch", "ovpn": "mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39"}, {"name": "xiubli", "ovpn": "root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n"}, {"name": "jafaj", "ovpn": "jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f"}, {"name": "jschmid", "ovpn": "jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef"}, {"name": "kachau", "ovpn": "kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe"}, {"name": "khiremat", "ovpn": "khiremat@kotresh-T490s 7o9Jj 2021-11-11T20:38:48.027 INFO:teuthology.task.ansible.out:enDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315"}, {"name": "kapandya", "ovpn": "kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67"}, {"name": "prasad", "ovpn": "prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89"}, {"name": "vikhyat", "ovpn": "vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f"}, {"name": "shraddhaag", "ovpn": "shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2"}, {"name": "gkyratsas", "ovpn": "gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7"}, {"name": "prateek", "ovpn": "prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67"}, {"name": "jelopez", "ovpn": "jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e"}, {"name": "dcain", "ovpn": "dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389"}, {"name": "sunny", "ovpn": "sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11"}, {"name": "xuxuehan", "ovpn": "xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n"}, {"name": "aclewett", "ovpn": "aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798"}, {"name": "cblum", "ovpn": "cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56"}, {"name": "mlatimer", "ovpn": "mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f"}, {"name": "shyamsundarr", "ovpn": "sh 2021-11-11T20:38:48.028 INFO:teuthology.task.ansible.out:yamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5"}, {"name": "shuzhenyi", "ovpn": "bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501"}, {"name": "mdipalma", "ovpn": "mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14"}, {"name": "kchang", "ovpn": "kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3"}, {"name": "vereddy", "ovpn": "vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2"}, {"name": "benhanokh", "ovpn": "gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5"}, {"name": "jsalomon", "ovpn": "jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0"}, {"name": "jolmomar", "ovpn": "jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640"}, {"name": "julpark", "ovpn": "julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8"}, {"name": "pdhange", "ovpn": "pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535"}, {"name": "dpivonka", "ovpn": "dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4"}, {"name": "adking", "ovpn": "adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625"}, {"name": "amathuri", "ovpn": "amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n"}, {"name": "nizamial09", "ovpn": "nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a 2021-11-11T20:38:48.030 INFO:teuthology.task.ansible.out:28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409"}, {"name": "dachary", "ovpn": "loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5"}, {"name": "amctagga", "ovpn": "amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1"}, {"name": "kevinz", "ovpn": "kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f"}, {"name": "liu-chunmei", "ovpn": "liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5 2021-11-11T20:38:48.030 INFO:teuthology.task.ansible.out:299298461a24cab483b9765293eafb2050c"}, {"name": "cyx1231st", "ovpn": "cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a"}, {"name": "sunnagar", "ovpn": "sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae"}, {"name": "lflores", "ovpn": "lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2"}, {"name": "jsawaya", "ovpn": "jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46"}, {"name": "markw8973240", "ovpn": "shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d"}, {"name": "luisrico", "ovpn": "lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347"}, {"name": "matan", "ovpn": "matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n"}, {"name": "ozeneva", "ovpn": "ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5"}, {"name": "sudhanaruna", "ovpn": "ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997"}, {"name": "zl31wang", "ovpn": " 2021-11-11T20:38:48.032 INFO:teuthology.task.ansible.out:zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2"}, {"name": "mayagilad", "ovpn": "mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0"}, {"name": "arjsharm", "ovpn": "arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39"}, {"name": "melissali", "ovpn": "melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07"}, {"name": "ionutbalutoiu", "ovpn": "ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1"}, {"name": "pdiazbou", "ovpn": "pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4"}, {"name": "xuwang", "ovpn": "xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8"}, {"name": "blaine", "ovpn": "blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547"}, {"name": "choffman", "ovpn": "choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c"}]}, "changed": false} 2021-11-11T20:38:48.056 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:38:48 +0000 (0:00:00.142) 0:01:08.312 ***** 2021-11-11T20:38:48.778 INFO:teuthology.task.ansible.out: TASK [users : Clone the keys repo] ********************************************* 2021-11-11T20:38:48.780 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"after": "d72e06d3c78c8d9639a8ca75effa043ee468910f", "attempts": 1, "before": "d72e06d3c78c8d9639a8ca75effa043ee468910f", "changed": false, "remote_url_changed": false} 2021-11-11T20:38:48.800 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:38:48 +0000 (0:00:00.744) 0:01:09.056 ***** 2021-11-11T20:38:48.808 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"after": "d72e06d3c78c8d9639a8ca75effa043ee468910f", "attempts": 1, "before": "d72e06d3c78c8d9639a8ca75effa043ee468910f", "changed": false, "remote_url_changed": false} 2021-11-11T20:38:48.833 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:38:48 +0000 (0:00:00.032) 0:01:09.089 ***** 2021-11-11T20:38:49.142 INFO:teuthology.task.ansible.out: TASK [users : Update authorized_keys using the keys repo] ********************** 2021-11-11T20:38:49.144 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'andrewschoen', 'ovpn': 'andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "andrewschoen", "ovpn": "andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDehvF7TbkfX9Ga4fNdq/j9GHu3c9cNk4iIHVs7e14j0h9DlV14aPyn/M0+z90PK+5p2uL9EvRzkbsPrfxpG3pxUa1pmVmKIPvBruA7ln8tExOVwO3XEsBG7b8VWUMlQmwRS9WEfkKAIHlr9cgIFGb1yrMW30RyL7keO1giotAq/wR9pMicyqYAbZOtI3MuPsDVTNxtyD3RwRUQicz6vNaK/RwfMCZVgdvoinrJ7VSyFVEiitzuQqMlrF935sfLvhuyxmwvJKfZI8QMEYcFbKd1cDnCk9n/PySNRY96gb41nb5pUk+0eaDu37jnmj7LLqoidD3UWUuHNnGBAAk+zNlD andrew.schoen@gmail.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDdkebJGXKQWBG/7wZG334P+EAdC+5VNMrjLjXhaZZbt3V9vKDQCodcU3zSOjMa/GDGg2WHQdO/vioCc6O/m61KTbgLDaaPSzHH16j/e1SaU8BIgKKU42AUBJzpPeaZ2wCC1i12Hq8ULwjEW29IDsnMortzlCSQmaTKc3DN2nCXZWQXA1rSiQ2AhIq5WNJY9S36lsg5tkex4xUajyNW4vPubLtkSv+WrzNK7ZJ8zfzP6eJEJ+Uj7CLdanBCgwoLGSPfGA5c/gyUT7mHibOUwvFB5sFVsozMPp2PXzUUIv70adJpOskLYfKY7TUSuaB4AUOPu6NVauzVIl3Tsn+uvHqvtn8KC3BPpyH7+JiFQWEt+aO+/ynprfoqCjktuf07SXo0ewoVYT2S++JEBa/N2hKRkzIoSPJ+Rvoazc8GZIjXpQRyhMdTV/LCSeyUTJ7lDQVrQVhUIHKnMm+hZmf4tLQKZvNAThO984CpY93ST5Ww07fjRLMXNZGuaz+zHKNexhWSPKr+ggChnZCOLl/F6zbDnBZoCaHR6hH4+QLz0K/FAgxN2Nn00RiOFkmIlzEioxMcwC/peTl3WzRfqPINdTftiTzcFi/c2QmlHhnxsueIwNhHOn4frkGkFmexdDrXRStERzSBJ13X4kWduPmQsn+bx5/7xwqC6M4bXpkYFOefOw== aschoen@redhat.com", "key_options": null, "keyfile": "/home/andrewschoen/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "andrewschoen", "validate_certs": true} 2021-11-11T20:38:49.174 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'andrewschoen', 'ovpn': 'andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "andrewschoen", "ovpn": "andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDehvF7TbkfX9Ga4fNdq/j9GHu3c9cNk4iIHVs7e14j0h9DlV14aPyn/M0+z90PK+5p2uL9EvRzkbsPrfxpG3pxUa1pmVmKIPvBruA7ln8tExOVwO3XEsBG7b8VWUMlQmwRS9WEfkKAIHlr9cgIFGb1yrMW30RyL7keO1giotAq/wR9pMicyqYAbZOtI3MuPsDVTNxtyD3RwRUQicz6vNaK/RwfMCZVgdvoinrJ7VSyFVEiitzuQqMlrF935sfLvhuyxmwvJKfZI8QMEYcFbKd1cDnCk9n/PySNRY96gb41nb5pUk+0eaDu37jnmj7LLqoidD3UWUuHNnGBAAk+zNlD andrew.schoen@gmail.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDdkebJGXKQWBG/7wZG334P+EAdC+5VNMrjLjXhaZZbt3V9vKDQCodcU3zSOjMa/GDGg2WHQdO/vioCc6O/m61KTbgLDaaPSzHH16j/e1SaU8BIgKKU42AUBJzpPeaZ2wCC1i12Hq8ULwjEW29IDsnMortzlCSQmaTKc3DN2nCXZWQXA1rSiQ2AhIq5WNJY9S36lsg5tkex4xUajyNW4vPubLtkSv+WrzNK7ZJ8zfzP6eJEJ+Uj7CLdanBCgwoLGSPfGA5c/gyUT7mHibOUwvFB5sFVsozMPp2PXzUUIv70adJpOskLYfKY7TUSuaB4AUOPu6NVauzVIl3Tsn+uvHqvtn8KC3BPpyH7+JiFQWEt+aO+/ynprfoqCjktuf07SXo0ewoVYT2S++JEBa/N2hKRkzIoSPJ+Rvoazc8GZIjXpQRyhMdTV/LCSeyUTJ7lDQVrQVhUIHKnMm+hZmf4tLQKZvNAThO984CpY93ST5Ww07fjRLMXNZGuaz+zHKNexhWSPKr+ggChnZCOLl/F6zbDnBZoCaHR6hH4+QLz0K/FAgxN2Nn00RiOFkmIlzEioxMcwC/peTl3WzRfqPINdTftiTzcFi/c2QmlHhnxsueIwNhHOn4frkGkFmexdDrXRStERzSBJ13X4kWduPmQsn+bx5/7xwqC6M4bXpkYFOefOw== aschoen@redhat.com", "key_options": null, "keyfile": "/home/andrewschoen/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "andrewschoen", "validate_certs": true} 2021-11-11T20:38:49.372 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'zack', 'ovpn': 'zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "zack", "ovpn": "zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBwJCoMbbxQrI9jM5I/lX0MYDB57LzYXDJTXqmLfJ8E5IHwQbn708/EFXAHuJlCyoJAuA8FaU/Y5/l2LBlVXwp9v+8ftk/D6AcsuQ+Hi0ZLCTvW10OUIS2cwX/of//JJIu6roy3r3SWwUcQvwjaZYLm7QkC+1StsLNMHdgiVRFiibqml72Q53VTtSTt6z2aXjrHzhpZq0hBK/13aknzAjHvGprQnchKVzhA/7A5pArF3CXpnI7aCwKvvjU0AcvqqD8WNLMpF4hFSoTK7xluKB+5UAJ36wefIO2KFM/zMwUtl4/aDhDdPeSaQRWq/q7viKLVNS0X9j2PjSRTfYhJNZr zack@zack-cerzas-macbook-pro.local", "key_options": null, "keyfile": "/home/zack/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "zack", "validate_certs": true} 2021-11-11T20:38:49.414 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'zack', 'ovpn': 'zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "zack", "ovpn": "zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBwJCoMbbxQrI9jM5I/lX0MYDB57LzYXDJTXqmLfJ8E5IHwQbn708/EFXAHuJlCyoJAuA8FaU/Y5/l2LBlVXwp9v+8ftk/D6AcsuQ+Hi0ZLCTvW10OUIS2cwX/of//JJIu6roy3r3SWwUcQvwjaZYLm7QkC+1StsLNMHdgiVRFiibqml72Q53VTtSTt6z2aXjrHzhpZq0hBK/13aknzAjHvGprQnchKVzhA/7A5pArF3CXpnI7aCwKvvjU0AcvqqD8WNLMpF4hFSoTK7xluKB+5UAJ36wefIO2KFM/zMwUtl4/aDhDdPeSaQRWq/q7viKLVNS0X9j2PjSRTfYhJNZr zack@zack-cerzas-macbook-pro.local", "key_options": null, "keyfile": "/home/zack/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "zack", "validate_certs": true} 2021-11-11T20:38:49.602 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kdreyer', 'ovpn': 'kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kdreyer", "ovpn": "kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4eHmz10szeHNS3dNejKokW85ksB+iR4HGOFsmQM11Ni68Nm5aqEKvkOZU8TpY92vpCQL0A68GlrXB845cACdyk6HUJYyNNNMC43l1FYWOwjMqQBSdj8W3VQDTA6eiG60mt5fgI8fyR38rKzIA1MnTBkSSjuh5kQVJ9bdEp3GuY5oc8vxDNBlGJ6LYnyEWt/pqL2J+mpjqnOjsC+EbE2exhP9O+mvzpQiyo/+dEN1COwX3//pNRXGfOSeOczHNsJE8Eu+j/n/BlW57++sJyFMkzS7bUxMSGM6quvjQZ7RT1c5JM6vLEiQyzQxoRgzY93h1yKlOstBi0NamtpqHQZGP kdreyer@mbp.ktdreyer.com", "key_options": null, "keyfile": "/home/kdreyer/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kdreyer", "validate_certs": true} 2021-11-11T20:38:49.634 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kdreyer', 'ovpn': 'kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kdreyer", "ovpn": "kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4eHmz10szeHNS3dNejKokW85ksB+iR4HGOFsmQM11Ni68Nm5aqEKvkOZU8TpY92vpCQL0A68GlrXB845cACdyk6HUJYyNNNMC43l1FYWOwjMqQBSdj8W3VQDTA6eiG60mt5fgI8fyR38rKzIA1MnTBkSSjuh5kQVJ9bdEp3GuY5oc8vxDNBlGJ6LYnyEWt/pqL2J+mpjqnOjsC+EbE2exhP9O+mvzpQiyo/+dEN1COwX3//pNRXGfOSeOczHNsJE8Eu+j/n/BlW57++sJyFMkzS7bUxMSGM6quvjQZ7RT1c5JM6vLEiQyzQxoRgzY93h1yKlOstBi0NamtpqHQZGP kdreyer@mbp.ktdreyer.com", "key_options": null, "keyfile": "/home/kdreyer/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kdreyer", "validate_certs": true} 2021-11-11T20:38:49.842 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dmick', 'ovpn': 'dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dmick", "ovpn": "dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC305rmql/Qm8Si+DFvAs77hSUZakwiXhhAhSetS89bWq3AdPQTL1MUXJHFm3AMzwzb00tndvsl1AUmFseXLfAi87daNcKJdwEj/JvXgcHKzAXOpVRfArHh08JnOumQiP4EoLYvq7rL+3LjEFURKOWJLLnGxh4E+iiJAnxLTG3WuUcnz9kXVTwtmIs60+2yH+gZJkkB04mnivPYjX46sHigoKgtlE2JnQfOU1lBkeZaBeUhoZX2rIKwRyGELPPcUBwHk10izyk6tePLAw9bSWeS9uEaa/m+qa8neJDyR4/hWQNR9yRXp0YjKkiKoFog+qtsBt3xBNzTPsFLxZ5zugmp dmick@angus\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC/onMoOCUhkp+UiJAq+ye4P70sfeV+uMhgWKLfLBh/1SYq1xxb9bSmjBnETO/9KvukRhzlReBQ7y+BKRhm037P7LuTUY4r0Y8JB6BXkjjCY1yXZJC6kvB7xFsvyiDum5rrdeWMQ/Rhh7YJpze5KY1nHLkyU9X7Ewak0kLEyduJEKekbsOom6l0wZ2ZYrBEKFWJXUDk5icUuWWg3mhKaKnLNWVg1lUw8jzYTJYfjR9GUgzZWXtzkAaI3ugpLOlQ+KEls03/WAFcWA9F9ExqzIsJpvEks27LeFBGVE3PjVNKMqqBZQlZ 2021-11-11T20:38:49.844 INFO:teuthology.task.ansible.out:tIEz1iguWJaWMi5lDIfXkvVOFabtIpX/BZRf dmick@dmick-MacBookAir\nssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOoIm6sOytKwqNf9FSQn0Olu72ZRDjAYeNFQVWTn2PIB ed25519-hplaptop", "key_options": null, "keyfile": "/home/dmick/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dmick", "validate_certs": true} 2021-11-11T20:38:49.874 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dmick', 'ovpn': 'dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dmick", "ovpn": "dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC305rmql/Qm8Si+DFvAs77hSUZakwiXhhAhSetS89bWq3AdPQTL1MUXJHFm3AMzwzb00tndvsl1AUmFseXLfAi87daNcKJdwEj/JvXgcHKzAXOpVRfArHh08JnOumQiP4EoLYvq7rL+3LjEFURKOWJLLnGxh4E+iiJAnxLTG3WuUcnz9kXVTwtmIs60+2yH+gZJkkB04mnivPYjX46sHigoKgtlE2JnQfOU1lBkeZaBeUhoZX2rIKwRyGELPPcUBwHk10izyk6tePLAw9bSWeS9uEaa/m+qa8neJDyR4/hWQNR9yRXp0YjKkiKoFog+qtsBt3xBNzTPsFLxZ5zugmp dmick@angus\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC/onMoOCUhkp+UiJAq+ye4P70sfeV+uMhgWKLfLBh/1SYq1xxb9bSmjBnETO/9KvukRhzlReBQ7y+BKRhm037P7LuTUY4r0Y8JB6BXkjjCY1yXZJC6kvB7xFsvyiDum5rrdeWMQ/Rhh7YJpze5KY1nHLkyU9X7Ewak0kLEyduJEKekbsOom6l0wZ2ZYrBEKFWJXUDk5icUuWWg3mhKaKnLNWVg1lUw8jzYTJYfjR9GUgzZWXtzkAaI3ugpLOlQ+KEls03/WAFcWA9F9ExqzIsJpvEks27LeFBGVE3PjVNKMqqBZQlZ 2021-11-11T20:38:49.876 INFO:teuthology.task.ansible.out:tIEz1iguWJaWMi5lDIfXkvVOFabtIpX/BZRf dmick@dmick-MacBookAir\nssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOoIm6sOytKwqNf9FSQn0Olu72ZRDjAYeNFQVWTn2PIB ed25519-hplaptop", "key_options": null, "keyfile": "/home/dmick/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dmick", "validate_certs": true} 2021-11-11T20:38:50.071 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'sage', 'ovpn': 'sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sage", "ovpn": "sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJ4z0gLwjexgyixdTuiwaxpPnq7w4FOKxHsQKKAHTlKRu11ynB+9APbtqaAYss8yDqHaHyewPdE+9JMYhZX8rUgcKv2H7jBCQ/Enfl0UyY643VQnuG8lr2Ipk6ELuL5NuuTb7Npo4zKyq/0649kPD46k/X/qLuKzhN49k50S3XJq5WwHi6CJvR8DzPOkjied0o1BHpfcCCIMorUcyfVxqmeu2m+uSE15ncQsd8BIvH4/YNoncUGYxqVhGB+FNUlWdhV0SPRW7V/gmyvFW5qdYX2QhNZAQHChdEH1E9AI7WiCFkyLRx39qYgXgqj8vOOnlwHgy/oJnpnu/Mlvj1uETX sage@montreal\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCVPfoCpv2/9uWCwEo4clWt3i1O8ZPpB2FcQbBLxY3iLQWZJDe74rNWPBj0cJVCytTHxYV0e5HxYyChOG60ca5odHlPNB+/Z4WLU9LVuHiCYsElivJlwvD+l9kzI+UMGJb3HCHqx2yIRGXpEkSMFR1+w5Oz9sInPD7JJmO8Hb9KkoeqZtkJL0aic/M5bDWBut6SSfYlmh8qnQYbkCDiXg/KtOcy1ZJXQ9ZLjsCbpSzeBNbdUOxEQ1+5vUe00vHpl0vOKtRrzgTne2ct1IxbSjdL16ZgHi5BQAREXc9UUG3z7NPOmnwkuKkkpyLmwcOusot5tYHrxks+5i6EoGEJrvJ3 sage@maetl", "key_options": null, "keyfile": "/home/sage/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sage", "validate_certs": true} 2021-11-11T20:38:50.113 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'sage', 'ovpn': 'sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sage", "ovpn": "sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJ4z0gLwjexgyixdTuiwaxpPnq7w4FOKxHsQKKAHTlKRu11ynB+9APbtqaAYss8yDqHaHyewPdE+9JMYhZX8rUgcKv2H7jBCQ/Enfl0UyY643VQnuG8lr2Ipk6ELuL5NuuTb7Npo4zKyq/0649kPD46k/X/qLuKzhN49k50S3XJq5WwHi6CJvR8DzPOkjied0o1BHpfcCCIMorUcyfVxqmeu2m+uSE15ncQsd8BIvH4/YNoncUGYxqVhGB+FNUlWdhV0SPRW7V/gmyvFW5qdYX2QhNZAQHChdEH1E9AI7WiCFkyLRx39qYgXgqj8vOOnlwHgy/oJnpnu/Mlvj1uETX sage@montreal\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCVPfoCpv2/9uWCwEo4clWt3i1O8ZPpB2FcQbBLxY3iLQWZJDe74rNWPBj0cJVCytTHxYV0e5HxYyChOG60ca5odHlPNB+/Z4WLU9LVuHiCYsElivJlwvD+l9kzI+UMGJb3HCHqx2yIRGXpEkSMFR1+w5Oz9sInPD7JJmO8Hb9KkoeqZtkJL0aic/M5bDWBut6SSfYlmh8qnQYbkCDiXg/KtOcy1ZJXQ9ZLjsCbpSzeBNbdUOxEQ1+5vUe00vHpl0vOKtRrzgTne2ct1IxbSjdL16ZgHi5BQAREXc9UUG3z7NPOmnwkuKkkpyLmwcOusot5tYHrxks+5i6EoGEJrvJ3 sage@maetl", "key_options": null, "keyfile": "/home/sage/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sage", "validate_certs": true} 2021-11-11T20:38:50.301 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dgalloway', 'ovpn': 'dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dgalloway", "ovpn": "dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n"}, "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAN3MlkYQNsm1uB6uJQ7hWJ1Ro9Gusrhi/b/KshPMFh51VWBQn71LdCCjdrsE6FWs5FYE/hr7MITJe+si7aZCmuf8U9tc+oiYh3spXtRhJVIM80FOVRXUmt2XR8I+oNbpcFDayYu956Volafksk3oYjsO356swj7+Q+AUQVF4bDJRAAAAFQCkB2YZKItSDL7XfAl6ZmJ21vqUMwAAAIEAsZspNIUwssVJGO4r3M3BECWklWLEFviIjGVKdPWulN10eRCczZrH7vMOs8coHKDQqYdS2RejlojT03FjFAqeBiTeAlcmgHoev+vqTg2Yhqx3gROzNc6NmG887owCOU4j7LJQP6SX8I8EpSXCMNOZliv2ZokjX2DuRsuIDxQdc84AAACBAMAp++fcqmGS7x7ic2x4rhyiYmdqlMIl8lrM3GkVR6qGuIVBvOGl38ieJKcTBLeRgyC04uIMfk8P9lLvZw3UF97qH3V0imKfq8aYnBGxCv+IjtdOrR5k4hrUW+P0qipvU+4dgHlpqwTKB4MtD9uDxvhENWvqcCnOS5rI6dddaZT6 dgallowa@dgallowa.csb\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKT1KilDopHJzDZ90QJx4+7WVIC4WNdvpD6VrS83B3PFOplNyXZLfDSDRZ4wEzIKLwtR9vLWWInVBNFU00W/2dObSC+l9GDzKsHtO3KSVmAjHEk6ZAJ5Mc3eUPR0lAywpFD8+yg+kDDIkRtCtoHDq+kFvNQutdvrMuhNWOZLyQ2N2XrpRJVD2kiJvaml6nQoZXMtVQGPVcYbn3TRed/wxT/han2u6j/wiElv4r9ty+9zi3P1MoNiALD53sD7wMr/5Si6PX+5zWcXTNcQmTuGpiLwGw0RxCwr4gtwqKBaNae2d1bwnHanLc1KvQEw29O6aBca2f7GWxFY6QtQV/mNft dgalloway@w541\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDoEI+g74GcJe/03nDypMzDWqg93K1h13xQPFvCIN7ElNZ+ulY5av5KLnv/gmLbYGRPlgxC7LNZXE7YQPUo2DfA8awnDkAkE/nWvL4fQjUxmaiPthjIWsKvk9EvPIqzLjDbwsiLiRPE0Sxka0qBa1GHU1QYN6cw7l9nWF0dG/TsKw+QmuArBKMgm9Th/QVLq1ARHC/07M1n195QvBjdrKSzOUl+2L1tADHg++sEWDq/MJMiylyNdxp79Wz7aTIlR6l8NV+Ega9LUb6B6xmO7k+J+/hJF7gYOeFl+YqnZttEdz+ 2021-11-11T20:38:50.303 INFO:teuthology.task.ansible.out:o8FbLjhTjOF9QkBsGWlrawXtQln/cVCryV+LAt57L dgalloway@p50", "key_options": null, "keyfile": "/home/dgalloway/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dgalloway", "validate_certs": true} 2021-11-11T20:38:50.343 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dgalloway', 'ovpn': 'dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dgalloway", "ovpn": "dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n"}, "key": "ssh-dss AAAAB3NzaC1kc3MAAACBAN3MlkYQNsm1uB6uJQ7hWJ1Ro9Gusrhi/b/KshPMFh51VWBQn71LdCCjdrsE6FWs5FYE/hr7MITJe+si7aZCmuf8U9tc+oiYh3spXtRhJVIM80FOVRXUmt2XR8I+oNbpcFDayYu956Volafksk3oYjsO356swj7+Q+AUQVF4bDJRAAAAFQCkB2YZKItSDL7XfAl6ZmJ21vqUMwAAAIEAsZspNIUwssVJGO4r3M3BECWklWLEFviIjGVKdPWulN10eRCczZrH7vMOs8coHKDQqYdS2RejlojT03FjFAqeBiTeAlcmgHoev+vqTg2Yhqx3gROzNc6NmG887owCOU4j7LJQP6SX8I8EpSXCMNOZliv2ZokjX2DuRsuIDxQdc84AAACBAMAp++fcqmGS7x7ic2x4rhyiYmdqlMIl8lrM3GkVR6qGuIVBvOGl38ieJKcTBLeRgyC04uIMfk8P9lLvZw3UF97qH3V0imKfq8aYnBGxCv+IjtdOrR5k4hrUW+P0qipvU+4dgHlpqwTKB4MtD9uDxvhENWvqcCnOS5rI6dddaZT6 dgallowa@dgallowa.csb\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKT1KilDopHJzDZ90QJx4+7WVIC4WNdvpD6VrS83B3PFOplNyXZLfDSDRZ4wEzIKLwtR9vLWWInVBNFU00W/2dObSC+l9GDzKsHtO3KSVmAjHEk6ZAJ5Mc3eUPR0lAywpFD8+yg+kDDIkRtCtoHDq+kFvNQutdvrMuhNWOZLyQ2N2XrpRJVD2kiJvaml6nQoZXMtVQGPVcYbn3TRed/wxT/han2u6j/wiElv4r9ty+9zi3P1MoNiALD53sD7wMr/5Si6PX+5zWcXTNcQmTuGpiLwGw0RxCwr4gtwqKBaNae2d1bwnHanLc1KvQEw29O6aBca2f7GWxFY6QtQV/mNft dgalloway@w541\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDoEI+g74GcJe/03nDypMzDWqg93K1h13xQPFvCIN7ElNZ+ulY5av5KLnv/gmLbYGRPlgxC7LNZXE7YQPUo2DfA8awnDkAkE/nWvL4fQjUxmaiPthjIWsKvk9EvPIqzLjDbwsiLiRPE0Sxka0qBa1GHU1QYN6cw7l9nWF0dG/TsKw+QmuArBKMgm9Th/QVLq1ARHC/07M1n195QvBjdrKSzOUl+2L1tADHg++sEWDq/MJMiylyNdxp79Wz7aTIlR6l8NV+Ega9LUb6B6xmO7k+J+/hJF7gYOeFl+YqnZttEdz+ 2021-11-11T20:38:50.345 INFO:teuthology.task.ansible.out:o8FbLjhTjOF9QkBsGWlrawXtQln/cVCryV+LAt57L dgalloway@p50", "key_options": null, "keyfile": "/home/dgalloway/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dgalloway", "validate_certs": true} 2021-11-11T20:38:50.530 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'akraitma', 'ovpn': 'akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "akraitma", "ovpn": "akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDo3gswIiZ2xPWMUQNjV1Gd4gYSxVpJM8Jb5pH1lqvoKAsmhVI30qTYxIaezzDWfSYwdkNYQAC9eM5v7EMQrOI7eZtTzCH+gbF79GciGCLL2bDze1I8hJ+jUu4bv+wnmvKsHyyOVBBJPSJ+eGFCzsu7vOiK4iFX0zgCXYwvP+A4Y/+kVR1en39gMtD6Hrw8ACBhxt+C2G2zc1807NRj3pIw1GvQILa9Sm3VKdNSrHzGB+MBooUoRmmjjzZ/bRwHPWVj+sX+xkoX4P2zwUcfMgZ/f4GgSgHyEvS3oxqbSaZp0pidZyoCaLRgqVPiUgXQEm0HA5PDDULAthQOfxQveKB/ akraitma@aklap", "key_options": null, "keyfile": "/home/akraitma/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "akraitma", "validate_certs": true} 2021-11-11T20:38:50.572 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'akraitma', 'ovpn': 'akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "akraitma", "ovpn": "akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDo3gswIiZ2xPWMUQNjV1Gd4gYSxVpJM8Jb5pH1lqvoKAsmhVI30qTYxIaezzDWfSYwdkNYQAC9eM5v7EMQrOI7eZtTzCH+gbF79GciGCLL2bDze1I8hJ+jUu4bv+wnmvKsHyyOVBBJPSJ+eGFCzsu7vOiK4iFX0zgCXYwvP+A4Y/+kVR1en39gMtD6Hrw8ACBhxt+C2G2zc1807NRj3pIw1GvQILa9Sm3VKdNSrHzGB+MBooUoRmmjjzZ/bRwHPWVj+sX+xkoX4P2zwUcfMgZ/f4GgSgHyEvS3oxqbSaZp0pidZyoCaLRgqVPiUgXQEm0HA5PDDULAthQOfxQveKB/ akraitma@aklap", "key_options": null, "keyfile": "/home/akraitma/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "akraitma", "validate_certs": true} 2021-11-11T20:38:50.769 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jcaratzas', 'ovpn': 'jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jcaratzas", "ovpn": "jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAV78mqEy/VkB2BthIE/m7iE5sbTjTRS0KuH7AB0KPvZBsDIw5cL8jKyxiaPC85+aOkBa4LrP0TUFBzN87Snt+PLMz2QHBOvlSTfUuRkHVo3vWJFzVuUSzzddkYds3hAxUriAHKv6CRSy1yFBxPQQHTSATUHKWqihMElbUCZrrrfH8A7J9vwixzCSLg4KbINjfa77rxefcZwugUqiYiVUWUSOb023f+QEaDm7OP+/uzz3SjF1Y0LMwa75yctttIfkRBKjY7uFCuZdKc3l2A/fdPxbFwlZmIox3LLngdTzHMVXGbVHqq9Ixo8FSAXiI1Buy2LTMoXMJwQYSvzsv7xNwujKVlUjoE9cOOIdzCC5AziwSiJB4+tNh73c5tRUzMMC98HLOhPJriSPvOt1pMEdWDX3/evcyxvRgJUKgYnmQG7n8v8O/CLyG0sAJDmGC3U9S/3fUDUjZIKWSsc85Q3xRaJwZJIS2VZwH0zG1Gmynih8PPq7VeDx7tKuNm4uTYT8= bigjust@desktop-jsho3l9.lan", "key_options": null, "keyfile": "/home/jcaratzas/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jcaratzas", "validate_certs": true} 2021-11-11T20:38:50.801 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jcaratzas', 'ovpn': 'jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jcaratzas", "ovpn": "jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAV78mqEy/VkB2BthIE/m7iE5sbTjTRS0KuH7AB0KPvZBsDIw5cL8jKyxiaPC85+aOkBa4LrP0TUFBzN87Snt+PLMz2QHBOvlSTfUuRkHVo3vWJFzVuUSzzddkYds3hAxUriAHKv6CRSy1yFBxPQQHTSATUHKWqihMElbUCZrrrfH8A7J9vwixzCSLg4KbINjfa77rxefcZwugUqiYiVUWUSOb023f+QEaDm7OP+/uzz3SjF1Y0LMwa75yctttIfkRBKjY7uFCuZdKc3l2A/fdPxbFwlZmIox3LLngdTzHMVXGbVHqq9Ixo8FSAXiI1Buy2LTMoXMJwQYSvzsv7xNwujKVlUjoE9cOOIdzCC5AziwSiJB4+tNh73c5tRUzMMC98HLOhPJriSPvOt1pMEdWDX3/evcyxvRgJUKgYnmQG7n8v8O/CLyG0sAJDmGC3U9S/3fUDUjZIKWSsc85Q3xRaJwZJIS2VZwH0zG1Gmynih8PPq7VeDx7tKuNm4uTYT8= bigjust@desktop-jsho3l9.lan", "key_options": null, "keyfile": "/home/jcaratzas/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jcaratzas", "validate_certs": true} 2021-11-11T20:38:51.009 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'trociny', 'ovpn': 'mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "trociny", "ovpn": "mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSc2rWtLtZM0CBWwaxhSfd0T++ZTX9c8kQrYT0rsQlJ6Hxj4c3+ypwh4iivdtFpyz4clxWkb/wimi48PvjNyZdudB9BeQUBeFfLiro3LnFEbKhM1RwRI89s34mRwmC0LLUZwcdX80zc6/fKuPzvBzR1PiY9tWRyIpYErQAHa2jEOE2MpFDHnlbXdfXFh5Tw3OmC1qBlQvgcrivA9lVaVlf+hfwmOQwXFIG3PCJLNlXDshCSUGAfYKBRvQaKS8sOf9rQaflPrwqJ8hDgVVQkpp69yVsGtahbBmjs6QS6C8EkGUFQmvrIcrQ8+5mRlRAmK2DMTP3cAcsYcMOJ+9KwP+H mgolub@zhuzha", "key_options": null, "keyfile": "/home/trociny/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "trociny", "validate_certs": true} 2021-11-11T20:38:51.020 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'smithfarm', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf', 'ovpn': 'smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007'}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf", "name": "smithfarm", "ovpn": "smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:38:51.041 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'trociny', 'ovpn': 'mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "trociny", "ovpn": "mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSc2rWtLtZM0CBWwaxhSfd0T++ZTX9c8kQrYT0rsQlJ6Hxj4c3+ypwh4iivdtFpyz4clxWkb/wimi48PvjNyZdudB9BeQUBeFfLiro3LnFEbKhM1RwRI89s34mRwmC0LLUZwcdX80zc6/fKuPzvBzR1PiY9tWRyIpYErQAHa2jEOE2MpFDHnlbXdfXFh5Tw3OmC1qBlQvgcrivA9lVaVlf+hfwmOQwXFIG3PCJLNlXDshCSUGAfYKBRvQaKS8sOf9rQaflPrwqJ8hDgVVQkpp69yVsGtahbBmjs6QS6C8EkGUFQmvrIcrQ8+5mRlRAmK2DMTP3cAcsYcMOJ+9KwP+H mgolub@zhuzha", "key_options": null, "keyfile": "/home/trociny/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "trociny", "validate_certs": true} 2021-11-11T20:38:51.052 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'smithfarm', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf', 'ovpn': 'smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007'}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf", "name": "smithfarm", "ovpn": "smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:38:51.249 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mbenjamin', 'ovpn': 'mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mbenjamin", "ovpn": "mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n"}, "key": "ssh-dss AAAAB3NzaC1kc3MAAACBANarZ+C62DPffRIhaCuCtXLChrfqwXgWqGHOLLkmPTtZ3/EE8ZiN+LY1ZcmP939AxXi8Oe+9LepwAdsoQKZctlC7XToyhjIk+rVSPyn3zl3BTC2AxfCKq2vqj65a1sc35Whrw7OaUa2fr5iHSazP6HuG01DhJ9l8HJsLKmiwWChZAAAAFQCNmunc7BNLWrqyrYwINTfscowkbQAAAIEAyYERcunZYJAZ9Eno18kM+ylq0jcQmnPqyIvssWXTttUZnaGQvbVxABeyr6ZKuUyZQfyetofquOGBKPBuzV/7posl5ShDuAcxbS1zhrmk9glTA1yFLvznDdWdLOWG06tjOc78G2AmldKHCdPsmgh2D7WyRr4QY4rcCvwKfZ6+eEYAAACBAJZrOlOlygfPZ4ZB0+EbdYACz1+L/iYNNAOBp51TRutNceLVrNJNOVdmCMLuF0A1go4kPSD/05aJOkOyW86gYLA5DXaW3961+kPInOFAe8OTyQYE7221ThYHRnCfVFjdknuCDomj69wk4SsR+ZPXvSXs1jSJWKhLfUSe9n4bmw7w anet2@secant.private.linuxbox.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5CMht/crEocrHqGjf0sxE9AhNcW3XBxwlniJwaBJLhekLQEQJoxAy2xUUmPYb2/e7rdmTBxzQipUaAGWVk1FHANac4cbgXzNT+9cnF7EIRkPa/x9jaKNiU0ozMhu/GOoeDXMrHIDgVGWPF3Ag/r17ORpYnuFTjsqoCWyh+mqskV+lmAp04rv2tvqouHtEs6gLebKNtB8c6twhrBKF62Ie+rSr3r7hQaqiZitHbg/isbU5RMBl6H48wHAVSK2/uTfMor 2021-11-11T20:38:51.250 INFO:teuthology.task.ansible.out:bnuEYRG8HvlnPRvQHscJYKiZSt/G7XqZ3x/7c4czEF+OpdQbAgg46vh4fNVMxZk9KHwEkOE06ROuwgAejJ mbenjamin@duel", "key_options": null, "keyfile": "/home/mbenjamin/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mbenjamin", "validate_certs": true} 2021-11-11T20:38:51.280 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mbenjamin', 'ovpn': 'mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mbenjamin", "ovpn": "mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n"}, "key": "ssh-dss AAAAB3NzaC1kc3MAAACBANarZ+C62DPffRIhaCuCtXLChrfqwXgWqGHOLLkmPTtZ3/EE8ZiN+LY1ZcmP939AxXi8Oe+9LepwAdsoQKZctlC7XToyhjIk+rVSPyn3zl3BTC2AxfCKq2vqj65a1sc35Whrw7OaUa2fr5iHSazP6HuG01DhJ9l8HJsLKmiwWChZAAAAFQCNmunc7BNLWrqyrYwINTfscowkbQAAAIEAyYERcunZYJAZ9Eno18kM+ylq0jcQmnPqyIvssWXTttUZnaGQvbVxABeyr6ZKuUyZQfyetofquOGBKPBuzV/7posl5ShDuAcxbS1zhrmk9glTA1yFLvznDdWdLOWG06tjOc78G2AmldKHCdPsmgh2D7WyRr4QY4rcCvwKfZ6+eEYAAACBAJZrOlOlygfPZ4ZB0+EbdYACz1+L/iYNNAOBp51TRutNceLVrNJNOVdmCMLuF0A1go4kPSD/05aJOkOyW86gYLA5DXaW3961+kPInOFAe8OTyQYE7221ThYHRnCfVFjdknuCDomj69wk4SsR+ZPXvSXs1jSJWKhLfUSe9n4bmw7w anet2@secant.private.linuxbox.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5CMht/crEocrHqGjf0sxE9AhNcW3XBxwlniJwaBJLhekLQEQJoxAy2xUUmPYb2/e7rdmTBxzQipUaAGWVk1FHANac4cbgXzNT+9cnF7EIRkPa/x9jaKNiU0ozMhu/GOoeDXMrHIDgVGWPF3Ag/r17ORpYnuFTjsqoCWyh+mqskV+lmAp04rv2tvqouHtEs6gLebKNtB8c6twhrBKF62Ie+rSr3r7hQaqiZitHbg/isbU5RMBl6H48wHAVSK2/uTfMor 2021-11-11T20:38:51.282 INFO:teuthology.task.ansible.out:bnuEYRG8HvlnPRvQHscJYKiZSt/G7XqZ3x/7c4czEF+OpdQbAgg46vh4fNVMxZk9KHwEkOE06ROuwgAejJ mbenjamin@duel", "key_options": null, "keyfile": "/home/mbenjamin/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mbenjamin", "validate_certs": true} 2021-11-11T20:38:51.477 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'aemerson', 'ovpn': 'aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "aemerson", "ovpn": "aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvG6+jM8aulc5CID/yE6AXPXczxq8tMgY+2sdo/ihWpgDZuFwGL4hwj8nyh5cTqZ2LEa4kRkUUIoH9avLCFN5pDBVvcj1/O06ZGnkq/hV6791UiPF5S5RGmz3MGxv0RDfjA0OLFav7u1s+ZJ6z2W1Tpy/W+g/fMYqbPc6rV+pm3Uo16+BWz/diZH8smqcty4chnnCy6gqOrcAxCZLpeyjX6Ydn3fnTLTAmhoQ9eUz7LjP4ZnFD5zL6jrSBlJ5WNTLevUzvS8eL7RuP8D3Qfn8ukIzPXG2vs594FhSc/HnvubKricvKMm0P0ki4Z0vM9VruQn7Tbeqdwpi1IfgyiAMX aemerson@work-yubi\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCxUDk9jEQOSrRwe0WMGBPw28fzgOhvqMgcG90SvuUjR20Rx3KUfy9JMxPJPesjBQz0xgH5adOVg2JQeLrt3uSJdIFNpqox8BBrS1u/bWT2dorQjLnFEhrtA8Ae/h2kwLQ8w3yYNWB/PxDSuMRnyH4G8EWtVydzQcV/QnNZ9Y6eBcHaI5C2yVnFl7Fi4zBYc2eeL8o8gWqmM6DobBCvVrxD0pCNAsLhOW6IA140BNGT9F/hQZYje9RQRWbFKh/iKiUhqFYCzLcxFfYXkw2HZMJA2p/bLmuc8ZbgYaIiU6b90kpfDB37Xw0S6toIj9E8h+E3nkNnwraCQcbralhz/bdz aemerson@seasalt\nssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv8/FJn7hz2WieoQFUGXS0bhwSwL1LCI9kkjWYNkWb8IJPrBetlkr5TsvHEKDPZGF5Sv8LsYA6njPnbOGyAHpYlnv/Gz4Hwq3hLgmXCs6JGWBQv8NS+pjwnG6Nh6JfFI0ANeg652Rk9Qt9uLYeKVKmHXXrVZAD4NMvJGfEvneQzuwwpylUmV80WAdaQw/H0EP4NScZhNhknwAs/+KPFZH+RFoUj/Cb3poIFjwyviIgyal5hjAEsoczCW3hAla/9IyC2VDS027WPiPFmBFaYK5lYYLStBQdRx+6amH76XScLFy0FEPLL5qiN0ClAptvh1oIURgipdXZwaxpf06AXG8qw== aemerson@electric", "key_options": null, "keyfile": "/home/aemerson/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "aemer 2021-11-11T20:38:51.479 INFO:teuthology.task.ansible.out:son", "validate_certs": true} 2021-11-11T20:38:51.488 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'amaredia', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour', 'ovpn': 'ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78'}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour", "name": "amaredia", "ovpn": "ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:38:51.489 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'tserlin', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet', 'ovpn': 'tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet", "name": "tserlin", "ovpn": "tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:38:51.510 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'aemerson', 'ovpn': 'aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "aemerson", "ovpn": "aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvG6+jM8aulc5CID/yE6AXPXczxq8tMgY+2sdo/ihWpgDZuFwGL4hwj8nyh5cTqZ2LEa4kRkUUIoH9avLCFN5pDBVvcj1/O06ZGnkq/hV6791UiPF5S5RGmz3MGxv0RDfjA0OLFav7u1s+ZJ6z2W1Tpy/W+g/fMYqbPc6rV+pm3Uo16+BWz/diZH8smqcty4chnnCy6gqOrcAxCZLpeyjX6Ydn3fnTLTAmhoQ9eUz7LjP4ZnFD5zL6jrSBlJ5WNTLevUzvS8eL7RuP8D3Qfn8ukIzPXG2vs594FhSc/HnvubKricvKMm0P0ki4Z0vM9VruQn7Tbeqdwpi1IfgyiAMX aemerson@work-yubi\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCxUDk9jEQOSrRwe0WMGBPw28fzgOhvqMgcG90SvuUjR20Rx3KUfy9JMxPJPesjBQz0xgH5adOVg2JQeLrt3uSJdIFNpqox8BBrS1u/bWT2dorQjLnFEhrtA8Ae/h2kwLQ8w3yYNWB/PxDSuMRnyH4G8EWtVydzQcV/QnNZ9Y6eBcHaI5C2yVnFl7Fi4zBYc2eeL8o8gWqmM6DobBCvVrxD0pCNAsLhOW6IA140BNGT9F/hQZYje9RQRWbFKh/iKiUhqFYCzLcxFfYXkw2HZMJA2p/bLmuc8ZbgYaIiU6b90kpfDB37Xw0S6toIj9E8h+E3nkNnwraCQcbralhz/bdz aemerson@seasalt\nssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv8/FJn7hz2WieoQFUGXS0bhwSwL1LCI9kkjWYNkWb8IJPrBetlkr5TsvHEKDPZGF5Sv8LsYA6njPnbOGyAHpYlnv/Gz4Hwq3hLgmXCs6JGWBQv8NS+pjwnG6Nh6JfFI0ANeg652Rk9Qt9uLYeKVKmHXXrVZAD4NMvJGfEvneQzuwwpylUmV80WAdaQw/H0EP4NScZhNhknwAs/+KPFZH+RFoUj/Cb3poIFjwyviIgyal5hjAEsoczCW3hAla/9IyC2VDS027WPiPFmBFaYK5lYYLStBQdRx+6amH76XScLFy0FEPLL5qiN0ClAptvh1oIURgipdXZwaxpf06AXG8qw== aemerson@electric", "key_options": null, "keyfile": "/home/aemerson/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "aemer 2021-11-11T20:38:51.512 INFO:teuthology.task.ansible.out:son", "validate_certs": true} 2021-11-11T20:38:51.521 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'amaredia', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour', 'ovpn': 'ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78'}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour", "name": "amaredia", "ovpn": "ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:38:51.532 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'tserlin', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet', 'ovpn': 'tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet", "name": "tserlin", "ovpn": "tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:38:51.740 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dis', 'ovpn': 'dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dis", "ovpn": "dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHk8c8wUTTYolOOmMlniNrsjfRyF9pCCnaEQ2DXR73P0hbFMme5Md5cF3PTtPIOHSNkWk/qg7s6gZD8LGyFmloO42RBgLsW1qM8RWTDbPy5EQdrjxL9byVzBBCDoQPIS2M+0XnhYa6CBj38Sw8XPlS41CebAJEOQeyLG2I6CAI0rTij6gsqDfALbIezd/eWvmbksKYM1Lb4Gvj+k/OL/xDZzIZHF/P+mK+trY45gsUHxKZamkeHyAT4f3kiNJA+UwzTWGhafy6UE7yGqhZjt9xOZSQGuwroL7EtAzntxayPRNxQPfGDV+fYYf0QBbTtHyJeikPUuw7aR1Z3KwuSmWD dis@zambezi", "key_options": null, "keyfile": "/home/dis/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dis", "validate_certs": true} 2021-11-11T20:38:51.762 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dis', 'ovpn': 'dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dis", "ovpn": "dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHk8c8wUTTYolOOmMlniNrsjfRyF9pCCnaEQ2DXR73P0hbFMme5Md5cF3PTtPIOHSNkWk/qg7s6gZD8LGyFmloO42RBgLsW1qM8RWTDbPy5EQdrjxL9byVzBBCDoQPIS2M+0XnhYa6CBj38Sw8XPlS41CebAJEOQeyLG2I6CAI0rTij6gsqDfALbIezd/eWvmbksKYM1Lb4Gvj+k/OL/xDZzIZHF/P+mK+trY45gsUHxKZamkeHyAT4f3kiNJA+UwzTWGhafy6UE7yGqhZjt9xOZSQGuwroL7EtAzntxayPRNxQPfGDV+fYYf0QBbTtHyJeikPUuw7aR1Z3KwuSmWD dis@zambezi", "key_options": null, "keyfile": "/home/dis/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dis", "validate_certs": true} 2021-11-11T20:38:51.968 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'gregf', 'ovpn': 'gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "gregf", "ovpn": "gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n"}, "key": "ssh-dss AAAAB3NzaC1kc3MAAACBANaS5gZUnIp5UL/PM3xOrQiNVxfona7sD1XEXGYvNUyoojPWWAvdQRwdA/Ay8JsAjKeWUt0IwfWaPfbEoXU1TKAI+yyv5WzQUZAKmBFuYM0CUQnKrk72Dp6n8P2tnYen0ect22xEiW2+Y0JZ3wfV3IB3BQPoVCC9FQ+P3LHhYii/AAAAFQCgSceiZFeNItwJ2rJeQgVVTQKljwAAAIEA0Gd2VNCbbQ48vAF+m/i/CeQHJ9Dom9IOBN3v8XnSlOhtNzY1pJ0NRzrFQURCCeVQJaywr0et5FCXIDVqTuR4qfCu5XXU1KHpn/AvHstOd93Y0XYMwSXfNszOtpH7w9SEm7uBfKnLQNF843/MHZL4ZIdfxk73YjfMjrYFcvJwvM0AAACBAI9Wi2wG9Rrdbrgk8tqbnKMxxJCYt/Vakn5mQSc/WHqMUn14+Kb7oBtfTsfPFNK952FzU09ptNQvNWTPNMniI8l8lQi2B+Z28XUlFLL76uxJXWko99xFpf/d1kFBL0M7GRGmvvrMfeTCI+WIqh2jBVqbl4o8kjNx4p0VWyoiWeFJ gregf@pudgy\nssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAwSe7szyqMTqKK3D/JsXIa1PtPf1p8Njk0sMDSsJPA0gBGysmSgstke9fglOgeNIbNOBNDOY3vltuNCgUiQgVWOYhUaVpj+2vtS 2021-11-11T20:38:51.970 INFO:teuthology.task.ansible.out:oAWpXMQbNaWPGKhbxvJWMCLVckLENyoUZUBKLmTvRyKDjjq8CJ1TnfA2Xfo3a796EIOtD6essU3lAVIfIUILGD+HqptiHSB0spUJUVRxxlO2xw9rLYij1eUPRHrr9xWvDamr5JtKVtCLOL3M9QlltHieKlbJSCZq/EVxW9NAGA6IiqOhw2E1k59oGZUq4Qg+YLgYkB80Ms7nb1xpjQ8txvY+73xL/8i7Nqmr7ekKJfyTpRe03nCQ== gfarnum@mozart.thompsonian.net\nssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAzqJQfdAKaggPDgSpk8P6QjLeyQYOz183fVSKnqAa5EeaCJozS1dn8z+iphswT3YL6wqIMqVHMfkSUcEeTBwZmvydaghOMJEIQSP1PCzH0V2XprELrEcBC3QTvaElPjjEYgBVUvEw8sL/RIQ7mgt9vxD0NcCDspsz35fmU222KzFxD41PZ5XzHQ3Lumwdax7mcpqrw8Aa5/tgcEqAIu54HYRf1Qmk6+ueMAvpwkX3eiSEtkVvuV1Kxpc3/0mOfJ7cn+eyPX3qUbo4oY76hIYseoUIwEaVE/pNW+pjwDSy4gSOcuxYDEGKzCwZPZeQtRQWzRt3YNafxi5xsByobGLlZw== gfarnum@GF-Macbook.local\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1ik3LbyKsPiGwdh+vGrKSYLvhMg7OkqMwDNYU5LqWBSotogxyGQJUuD80ZavxmmdXC4qScddHtWOJjtyX1CTSDUe7INpSgGPGnmSExVFpcYRnmxHAB+9M/c9VorUkQ8IOKLmRO/BdYma3Rl9NzhHI73yd4zJlx1K4NuQnhnj0Z4yXO+k44RkBWDdMicBB5lBP0hETaH2Iq1u5Dai3u6N5N9bSPXv2PGKvvU395K0Wts2CxMhOBc7LAixysNzNWLrnG1ghS4jPVG2TIHOElP5lIXUv1Wlkool5rTUWgnVY/7loWkDus2JryDM7hLGFXqgT7HHcpI4gSITqRhWxzsUF gregf@fedoragreg.localdomain", "key_options": null, "keyfile": "/home/gregf/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "gregf", "validate_certs": true} 2021-11-11T20:38:52.000 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'gregf', 'ovpn': 'gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "gregf", "ovpn": "gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n"}, "key": "ssh-dss AAAAB3NzaC1kc3MAAACBANaS5gZUnIp5UL/PM3xOrQiNVxfona7sD1XEXGYvNUyoojPWWAvdQRwdA/Ay8JsAjKeWUt0IwfWaPfbEoXU1TKAI+yyv5WzQUZAKmBFuYM0CUQnKrk72Dp6n8P2tnYen0ect22xEiW2+Y0JZ3wfV3IB3BQPoVCC9FQ+P3LHhYii/AAAAFQCgSceiZFeNItwJ2rJeQgVVTQKljwAAAIEA0Gd2VNCbbQ48vAF+m/i/CeQHJ9Dom9IOBN3v8XnSlOhtNzY1pJ0NRzrFQURCCeVQJaywr0et5FCXIDVqTuR4qfCu5XXU1KHpn/AvHstOd93Y0XYMwSXfNszOtpH7w9SEm7uBfKnLQNF843/MHZL4ZIdfxk73YjfMjrYFcvJwvM0AAACBAI9Wi2wG9Rrdbrgk8tqbnKMxxJCYt/Vakn5mQSc/WHqMUn14+Kb7oBtfTsfPFNK952FzU09ptNQvNWTPNMniI8l8lQi2B+Z28XUlFLL76uxJXWko99xFpf/d1kFBL0M7GRGmvvrMfeTCI+WIqh2jBVqbl4o8kjNx4p0VWyoiWeFJ gregf@pudgy\nssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAwSe7szyqMTqKK3D/JsXIa1PtPf1p8Njk0sMDSsJPA0gBGysmSgstke9fglOgeNIbNOBNDOY3vltuNCgUiQgVWOYhUaVpj+2vtS 2021-11-11T20:38:52.002 INFO:teuthology.task.ansible.out:oAWpXMQbNaWPGKhbxvJWMCLVckLENyoUZUBKLmTvRyKDjjq8CJ1TnfA2Xfo3a796EIOtD6essU3lAVIfIUILGD+HqptiHSB0spUJUVRxxlO2xw9rLYij1eUPRHrr9xWvDamr5JtKVtCLOL3M9QlltHieKlbJSCZq/EVxW9NAGA6IiqOhw2E1k59oGZUq4Qg+YLgYkB80Ms7nb1xpjQ8txvY+73xL/8i7Nqmr7ekKJfyTpRe03nCQ== gfarnum@mozart.thompsonian.net\nssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAzqJQfdAKaggPDgSpk8P6QjLeyQYOz183fVSKnqAa5EeaCJozS1dn8z+iphswT3YL6wqIMqVHMfkSUcEeTBwZmvydaghOMJEIQSP1PCzH0V2XprELrEcBC3QTvaElPjjEYgBVUvEw8sL/RIQ7mgt9vxD0NcCDspsz35fmU222KzFxD41PZ5XzHQ3Lumwdax7mcpqrw8Aa5/tgcEqAIu54HYRf1Qmk6+ueMAvpwkX3eiSEtkVvuV1Kxpc3/0mOfJ7cn+eyPX3qUbo4oY76hIYseoUIwEaVE/pNW+pjwDSy4gSOcuxYDEGKzCwZPZeQtRQWzRt3YNafxi5xsByobGLlZw== gfarnum@GF-Macbook.local\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1ik3LbyKsPiGwdh+vGrKSYLvhMg7OkqMwDNYU5LqWBSotogxyGQJUuD80ZavxmmdXC4qScddHtWOJjtyX1CTSDUe7INpSgGPGnmSExVFpcYRnmxHAB+9M/c9VorUkQ8IOKLmRO/BdYma3Rl9NzhHI73yd4zJlx1K4NuQnhnj0Z4yXO+k44RkBWDdMicBB5lBP0hETaH2Iq1u5Dai3u6N5N9bSPXv2PGKvvU395K0Wts2CxMhOBc7LAixysNzNWLrnG1ghS4jPVG2TIHOElP5lIXUv1Wlkool5rTUWgnVY/7loWkDus2JryDM7hLGFXqgT7HHcpI4gSITqRhWxzsUF gregf@fedoragreg.localdomain", "key_options": null, "keyfile": "/home/gregf/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "gregf", "validate_certs": true} 2021-11-11T20:38:52.197 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'joshd', 'ovpn': 'jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "joshd", "ovpn": "jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgS5EfJyOxBkl/TtJikfb38rm6muQo7MwYufkLSBw9W2jFzt/ZMbVXyVsdZM4I6hWfZm1/q60UcER4yoA1xwYGOTmbFXrLUqqKxKTnCdyejuQ3miB7aiuisWO4vjb2+ppBwL9Mx775DPeQJehFyFK0qgh33tZO/9LWWYy7NOZdA9L2ei1PgMWa0VBeijcDR5GQSXzh0Yi+h/Ezu8jevu+NkW2IUJF6zPGjd9M9lAy7KG2dQjqqCX4XWmUS+MHJS6f9bsC5i39GWy29FvX/qFEArOMzuYRIFWE8iza95KSBxptb+Hw8PFBjQ1kTh6nC5ZV2X3ETcoj8fToTcgW3dTRF jdurgin@new-angeles", "key_options": null, "keyfile": "/home/joshd/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "joshd", "validate_certs": true} 2021-11-11T20:38:52.229 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'joshd', 'ovpn': 'jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "joshd", "ovpn": "jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgS5EfJyOxBkl/TtJikfb38rm6muQo7MwYufkLSBw9W2jFzt/ZMbVXyVsdZM4I6hWfZm1/q60UcER4yoA1xwYGOTmbFXrLUqqKxKTnCdyejuQ3miB7aiuisWO4vjb2+ppBwL9Mx775DPeQJehFyFK0qgh33tZO/9LWWYy7NOZdA9L2ei1PgMWa0VBeijcDR5GQSXzh0Yi+h/Ezu8jevu+NkW2IUJF6zPGjd9M9lAy7KG2dQjqqCX4XWmUS+MHJS6f9bsC5i39GWy29FvX/qFEArOMzuYRIFWE8iza95KSBxptb+Hw8PFBjQ1kTh6nC5ZV2X3ETcoj8fToTcgW3dTRF jdurgin@new-angeles", "key_options": null, "keyfile": "/home/joshd/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "joshd", "validate_certs": true} 2021-11-11T20:38:52.427 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'cmeno', 'ovpn': 'cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "cmeno", "ovpn": "cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed"}, "key": "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHT7CHaceRLB5mJv9SYKog1wBXsavzH3G3WCBEvVyK76JIslDudZm2ycJv66r+VlGaD8sFI5Mb/KKFMKcKesLRX+wHqZg9MlkoJgm7WGoRQ5I0M1Vt7TL3uJkVCaGZHtO7BXqzTx53C1AaJ1HN+DG2MGy5mm6tHwL901UVqG4EzjBnjxA== christinameno@boxtop", "key_options": null, "keyfile": "/home/cmeno/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "cmeno", "validate_certs": true} 2021-11-11T20:38:52.449 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'cmeno', 'ovpn': 'cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "cmeno", "ovpn": "cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed"}, "key": "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHT7CHaceRLB5mJv9SYKog1wBXsavzH3G3WCBEvVyK76JIslDudZm2ycJv66r+VlGaD8sFI5Mb/KKFMKcKesLRX+wHqZg9MlkoJgm7WGoRQ5I0M1Vt7TL3uJkVCaGZHtO7BXqzTx53C1AaJ1HN+DG2MGy5mm6tHwL901UVqG4EzjBnjxA== christinameno@boxtop", "key_options": null, "keyfile": "/home/cmeno/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "cmeno", "validate_certs": true} 2021-11-11T20:38:52.676 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ivancich', 'ovpn': 'ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ivancich", "ovpn": "ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDkIZyWnJXYv/z3EVeKHatjFUAdaomIMqzvFLrA9VskeM0Ctv8a83Wk3BbYvTFQWALjZjS2HVYzi5p6kTQ4qxXlcO7HxRw0j4otNauilvgIc4Lu5N4pUtYf0piEo+nrXz+WhELfmJ8t1Yx00uGEl9pOF04SOuXtpwwrCFAO6WMCcjDuYhoI2VBKO3W/OWcIuQ8ZY3j4iDdofFOUGyyoYVyOi/HILKjluYB+ues5+wBEkDuoi3MafCnIj8M0O5fU8/JoOTksOu779UN6blPiNdEgnwD00jA+RWYY6b12ZVkiF4rUVGCiiKKUwOOWkRhkA5OEcBgZ8cGWFgGfZM1GJA// ivancich@sleepy", "key_options": null, "keyfile": "/home/ivancich/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ivancich", "validate_certs": true} 2021-11-11T20:38:52.687 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ivancich', 'ovpn': 'ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ivancich", "ovpn": "ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDkIZyWnJXYv/z3EVeKHatjFUAdaomIMqzvFLrA9VskeM0Ctv8a83Wk3BbYvTFQWALjZjS2HVYzi5p6kTQ4qxXlcO7HxRw0j4otNauilvgIc4Lu5N4pUtYf0piEo+nrXz+WhELfmJ8t1Yx00uGEl9pOF04SOuXtpwwrCFAO6WMCcjDuYhoI2VBKO3W/OWcIuQ8ZY3j4iDdofFOUGyyoYVyOi/HILKjluYB+ues5+wBEkDuoi3MafCnIj8M0O5fU8/JoOTksOu779UN6blPiNdEgnwD00jA+RWYY6b12ZVkiF4rUVGCiiKKUwOOWkRhkA5OEcBgZ8cGWFgGfZM1GJA// ivancich@sleepy", "key_options": null, "keyfile": "/home/ivancich/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ivancich", "validate_certs": true} 2021-11-11T20:38:52.905 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'yuriw', 'ovpn': 'yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "yuriw", "ovpn": "yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCw3MhMNDYjkiQADoC1mg9ga0Yj174FHIFt4RQIC8qK+6cB0N3CM5R4JwP8pR4tYkVFWmnYRScYK5VVdC873Ac2BzYxmse97zLJTbUnYWTdY+OFi1FUozsy3PwtiNHSd48Z/PBvkinQIey2QlMKaW3/yz9Of/Yfm4prxg9lBsd6/tOOFj3AsqIQ1OVhMfgeVEkdCL7AAbEYyMxePZ3UFrDwj05HDLgmFzx1xsNJFOuBwLfU/QNHoXJQrcVy0rZq9fB5rg74iz1JZs0uDWocQQK4Nd3Rv+wRGBuew2fmkrS2bQWXzMoSL+/i7pbf74LKcbWAE0wXEImqQeREHxq4ZZ5J yuriweinstein@Yuris-MacBook-Pro.local\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJUmq7btpBMjjODHXp1888xoBuX+rhiZg3ufuo69RgDxUMymJn3CPJ/5NGXoZ3SGAWKk0w0Dx6K/VO4y92mAV+cO81y8wZR4J3fPV3IbWRjN0X3IFoAvJi5n9v5hpG7z3RZCGmQnOZ99m1i9dge/3S8NpuPLIvYkSo9z00d+mNP6D8rGnqOmEBfEhUQXLeH7rbPwaMWPgB/hrpiPCyLR/+sGC5bq9PAsKnocqZ12cnvCtYxpnA7et6nL15ftEpUrKYAmRDxotuoUcsllP66GSszjbrgN6OBNDSqpGt/+RSJJwzZEGQ073gJuMveKy9tvD/PyEttjvVKL2KsBuXGSuL ubuntu@yw\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDmXWgRRes8bzekTH2fTcxdAAquFn1SjPhws1Z9 2021-11-11T20:38:52.907 INFO:teuthology.task.ansible.out:CkxNaqKe25XnzPehqrhg9etouUwvORQHdDJdTlmhTSYtn588Omq+1bvniab0KzAYgGZsWdPma/3E9qF7ChXIMik9Umip0drzYTaexCuC/Nu46brTxlnL0vaxnnVjDwmHC0CEJs4qN4e2wSlsA0uBDRWjxWFMdTUgSzII5xasHypUYZ0y3MNdbfY+SVF69DEDH8EOQMGYm8JQXNJFloQvmvLUFqT/63FsdpzRVTph4A8hvI1f8i6ULb7fPQN5ZS62wdXd/KvrBXrCvhOi7R63RM60mkMRXm1XajTGmCKiikfiqgTr yuriw@Yuris-MacBook-Air.local\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC41kRr6B/RC32DrQKRByalH832H6BpCQF9XdTvgYi4+LJt+kAuIGLDF4xm4uApTmWRrLspWm0Kksc0ger03uNx5qJqMEUnJvjxJnSDC0/XyxWapVxIviFkvv0BjbhVNOOaMU55ywrZ9hHgMdaFwuMuCOaB7f9Tm6IUXm2k82OXueeIG23Kziq+IQBysWOjvNUy/LLpl7l4+xOkhNUupUiQCqHvFH98VUdYWkHeyamafGVZmYbFLrOXgmviKo0u8LPxcT7psKLk0yFG7fG0DSak6DDdKJqb+G/NqMSPxRSAI4FbXQ6UOZb32Ozr1Sql9yfOXEYWZuatn7W5F8DjSv3p yuriw@yuriw-RH", "key_options": null, "keyfile": "/home/yuriw/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "yuriw", "validate_certs": true} 2021-11-11T20:38:52.916 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'yuriw', 'ovpn': 'yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "yuriw", "ovpn": "yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCw3MhMNDYjkiQADoC1mg9ga0Yj174FHIFt4RQIC8qK+6cB0N3CM5R4JwP8pR4tYkVFWmnYRScYK5VVdC873Ac2BzYxmse97zLJTbUnYWTdY+OFi1FUozsy3PwtiNHSd48Z/PBvkinQIey2QlMKaW3/yz9Of/Yfm4prxg9lBsd6/tOOFj3AsqIQ1OVhMfgeVEkdCL7AAbEYyMxePZ3UFrDwj05HDLgmFzx1xsNJFOuBwLfU/QNHoXJQrcVy0rZq9fB5rg74iz1JZs0uDWocQQK4Nd3Rv+wRGBuew2fmkrS2bQWXzMoSL+/i7pbf74LKcbWAE0wXEImqQeREHxq4ZZ5J yuriweinstein@Yuris-MacBook-Pro.local\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJUmq7btpBMjjODHXp1888xoBuX+rhiZg3ufuo69RgDxUMymJn3CPJ/5NGXoZ3SGAWKk0w0Dx6K/VO4y92mAV+cO81y8wZR4J3fPV3IbWRjN0X3IFoAvJi5n9v5hpG7z3RZCGmQnOZ99m1i9dge/3S8NpuPLIvYkSo9z00d+mNP6D8rGnqOmEBfEhUQXLeH7rbPwaMWPgB/hrpiPCyLR/+sGC5bq9PAsKnocqZ12cnvCtYxpnA7et6nL15ftEpUrKYAmRDxotuoUcsllP66GSszjbrgN6OBNDSqpGt/+RSJJwzZEGQ073gJuMveKy9tvD/PyEttjvVKL2KsBuXGSuL ubuntu@yw\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDmXWgRRes8bzekTH2fTcxdAAquFn1SjPhws1Z9 2021-11-11T20:38:52.918 INFO:teuthology.task.ansible.out:CkxNaqKe25XnzPehqrhg9etouUwvORQHdDJdTlmhTSYtn588Omq+1bvniab0KzAYgGZsWdPma/3E9qF7ChXIMik9Umip0drzYTaexCuC/Nu46brTxlnL0vaxnnVjDwmHC0CEJs4qN4e2wSlsA0uBDRWjxWFMdTUgSzII5xasHypUYZ0y3MNdbfY+SVF69DEDH8EOQMGYm8JQXNJFloQvmvLUFqT/63FsdpzRVTph4A8hvI1f8i6ULb7fPQN5ZS62wdXd/KvrBXrCvhOi7R63RM60mkMRXm1XajTGmCKiikfiqgTr yuriw@Yuris-MacBook-Air.local\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC41kRr6B/RC32DrQKRByalH832H6BpCQF9XdTvgYi4+LJt+kAuIGLDF4xm4uApTmWRrLspWm0Kksc0ger03uNx5qJqMEUnJvjxJnSDC0/XyxWapVxIviFkvv0BjbhVNOOaMU55ywrZ9hHgMdaFwuMuCOaB7f9Tm6IUXm2k82OXueeIG23Kziq+IQBysWOjvNUy/LLpl7l4+xOkhNUupUiQCqHvFH98VUdYWkHeyamafGVZmYbFLrOXgmviKo0u8LPxcT7psKLk0yFG7fG0DSak6DDdKJqb+G/NqMSPxRSAI4FbXQ6UOZb32Ozr1Sql9yfOXEYWZuatn7W5F8DjSv3p yuriw@yuriw-RH", "key_options": null, "keyfile": "/home/yuriw/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "yuriw", "validate_certs": true} 2021-11-11T20:38:53.146 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'tamil', 'ovpn': 'tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "tamil", "ovpn": "tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCxrS30JmTZbnBd73s9PGZYxCkWH3u4En9Q0bYJan45W+j+y5vMzaNUVsp8We1tphqPQ9B0ygi80OxbHWu5gyv4g+M/JFtrnxv3QgBCYWeBWfh652Vbdlk1thl6btFnHUhVMSn5tkdnFxyVDJHIROoVXkHRaCNAUUy7tB60IhPAMomNcCGu9R8+opONFLXKTIfkYQ1cGbysr90DMs8ASEkvxdv40hIWhK4ufvL7CFFtgXuoC0c1WedBFOkr5LDegzL13xN97K9o5J8CAU0MUKk6H8KNKbQuNq1zANwymMFY599GIlxwai8iluJgfOObDRdCDmKM2f9RAEUXQM7oYoH7 tamil@tamil-VirtualBox\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvwiWVhNLeOLg72cpVIwbUYnmFh+Cqf/mE7Rc/Xci91on3GPrq++dC8t2GhTB5Zyq46EE27SmUkiZpHRnz7xwiJKM/tTgNEkffXDK3j0pA2E22iqV9FpfSA9Zdi1Puq7CPT7/rix4rF3SPvMtg/R1fkyornLFdzhJiMExCKJXGwf+QsKk4NVnr+nO3EEkKOoP1wio1bceZ53rCt9Tyu8NawAB2nL2fvtBPQDAriQ8bLX69XLbOfhFI6GJo815/j6CQxNmvA0l5ABJc8+A71jZmndzumFRgqBuhiCM5ohsGR0JGahUQjFEA4quNTx8EWTGRsqhAYszzpqJzuN+Xb6tJ tamil@macbookair\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDEPzEyDGI7poyXVPG70vgQpvlU/lTt6Vvv5CoHWJoSrHklDdJEqekhCB87BX4DTrqbb5xoM7yfxle/z9UzgVwSUWqsPdZHA4EbrNFRKFxi/gc2YrygVSjumTyzf9WJJ5vCZvTQexOj7rjWVnJkjXHKMXZXIAQN/bJf6J21t+4A3XLrl30/4XXyq664NONUNjBHxHtlbS2LM6Th/hK4B23oj2BugX2j/50vWgd9H36bPr8+JGskOi6MH2ndZv5pYtLznFzQmsbixjaN8kCuyQdlRCOndPEFoHFhDla2MeB+h3XQLDXW7J1TR2URFSfcNq5ndDS4i0b+t75Fjhb7AtLT tamil@ubuntu-delldesktop\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6tJ9y3solDCzFdXcczO8GJEwMqMYXWU5kuMz0rr/US6F72j0spSNmwGemMOZE9s7KtHgior/Ufw8/vMRkRhoBY 2021-11-11T20:38:53.148 INFO:teuthology.task.ansible.out:B0C5KT9utGNqrtDdvH98h43XnKZ3K/yaHv/T+u2EYWNgHtOXMtgIVFnGwSD7RzF6TbwSfurprTV6ZUyyOxRcmHnkmwpbNwiwIoXB03k2PITdYCC3lEBEY0OC+vAW/E3+bFbV3RSEA0Cm31jmMyZyPeYiXiEnPXUIClsoDj34NwJTIgctwIsqM4eYUBg3pHliWYKEWDs55HZYoj3W/6Jw7MASzyAYFSmP8rQu8iS7tO1iEBRGOXFNxHirGQ1WqUn tmuthamizhan@MacBook-Air.local", "key_options": null, "keyfile": "/home/tamil/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "tamil", "validate_certs": true} 2021-11-11T20:38:53.177 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'tamil', 'ovpn': 'tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "tamil", "ovpn": "tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCxrS30JmTZbnBd73s9PGZYxCkWH3u4En9Q0bYJan45W+j+y5vMzaNUVsp8We1tphqPQ9B0ygi80OxbHWu5gyv4g+M/JFtrnxv3QgBCYWeBWfh652Vbdlk1thl6btFnHUhVMSn5tkdnFxyVDJHIROoVXkHRaCNAUUy7tB60IhPAMomNcCGu9R8+opONFLXKTIfkYQ1cGbysr90DMs8ASEkvxdv40hIWhK4ufvL7CFFtgXuoC0c1WedBFOkr5LDegzL13xN97K9o5J8CAU0MUKk6H8KNKbQuNq1zANwymMFY599GIlxwai8iluJgfOObDRdCDmKM2f9RAEUXQM7oYoH7 tamil@tamil-VirtualBox\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvwiWVhNLeOLg72cpVIwbUYnmFh+Cqf/mE7Rc/Xci91on3GPrq++dC8t2GhTB5Zyq46EE27SmUkiZpHRnz7xwiJKM/tTgNEkffXDK3j0pA2E22iqV9FpfSA9Zdi1Puq7CPT7/rix4rF3SPvMtg/R1fkyornLFdzhJiMExCKJXGwf+QsKk4NVnr+nO3EEkKOoP1wio1bceZ53rCt9Tyu8NawAB2nL2fvtBPQDAriQ8bLX69XLbOfhFI6GJo815/j6CQxNmvA0l5ABJc8+A71jZmndzumFRgqBuhiCM5ohsGR0JGahUQjFEA4quNTx8EWTGRsqhAYszzpqJzuN+Xb6tJ tamil@macbookair\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDEPzEyDGI7poyXVPG70vgQpvlU/lTt6Vvv5CoHWJoSrHklDdJEqekhCB87BX4DTrqbb5xoM7yfxle/z9UzgVwSUWqsPdZHA4EbrNFRKFxi/gc2YrygVSjumTyzf9WJJ5vCZvTQexOj7rjWVnJkjXHKMXZXIAQN/bJf6J21t+4A3XLrl30/4XXyq664NONUNjBHxHtlbS2LM6Th/hK4B23oj2BugX2j/50vWgd9H36bPr8+JGskOi6MH2ndZv5pYtLznFzQmsbixjaN8kCuyQdlRCOndPEFoHFhDla2MeB+h3XQLDXW7J1TR2URFSfcNq5ndDS4i0b+t75Fjhb7AtLT tamil@ubuntu-delldesktop\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6tJ9y3solDCzFdXcczO8GJEwMqMYXWU5kuMz0rr/US6F72j0spSNmwGemMOZE9s7KtHgior/Ufw8/vMRkRhoBY 2021-11-11T20:38:53.179 INFO:teuthology.task.ansible.out:B0C5KT9utGNqrtDdvH98h43XnKZ3K/yaHv/T+u2EYWNgHtOXMtgIVFnGwSD7RzF6TbwSfurprTV6ZUyyOxRcmHnkmwpbNwiwIoXB03k2PITdYCC3lEBEY0OC+vAW/E3+bFbV3RSEA0Cm31jmMyZyPeYiXiEnPXUIClsoDj34NwJTIgctwIsqM4eYUBg3pHliWYKEWDs55HZYoj3W/6Jw7MASzyAYFSmP8rQu8iS7tO1iEBRGOXFNxHirGQ1WqUn tmuthamizhan@MacBook-Air.local", "key_options": null, "keyfile": "/home/tamil/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "tamil", "validate_certs": true} 2021-11-11T20:38:53.385 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jowilkin', 'ovpn': 'jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jowilkin", "ovpn": "jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYGVqKZgLx2LJ49x0kGo/Z5r8MVefSUCYhhrK/DK8ht+YK1AqLXErjAedgpVz0WuwirRjh5YkiXGMYiOdkMUYWGk5nwCHEKnnCpne7Xh1OOqge/QGHlUigWVH8VIHBsZ+wTW25mqOrgDSGB8+Qk7Dr2o1sLGUHcx7vITDWOrer4KjF2+H5TQH/7irUVORu6fzeGKtGyOrTwQ+ZT1+fuhfznDULFGyvKGdcXxxNOg+21Fssk2nCNTlgRCGjHSrfelfuMT7QEt+9puWd7MWXYQg7xx8jkJGMpEQRFxJTxnfUlUXf3x2lc9Q/SvKfpVMtc0N+bk2F9+euyvRvjtn4LJNx jowilkin@jowilkin.wilkins", "key_options": null, "keyfile": "/home/jowilkin/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jowilkin", "validate_certs": true} 2021-11-11T20:38:53.407 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jowilkin', 'ovpn': 'jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jowilkin", "ovpn": "jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYGVqKZgLx2LJ49x0kGo/Z5r8MVefSUCYhhrK/DK8ht+YK1AqLXErjAedgpVz0WuwirRjh5YkiXGMYiOdkMUYWGk5nwCHEKnnCpne7Xh1OOqge/QGHlUigWVH8VIHBsZ+wTW25mqOrgDSGB8+Qk7Dr2o1sLGUHcx7vITDWOrer4KjF2+H5TQH/7irUVORu6fzeGKtGyOrTwQ+ZT1+fuhfznDULFGyvKGdcXxxNOg+21Fssk2nCNTlgRCGjHSrfelfuMT7QEt+9puWd7MWXYQg7xx8jkJGMpEQRFxJTxnfUlUXf3x2lc9Q/SvKfpVMtc0N+bk2F9+euyvRvjtn4LJNx jowilkin@jowilkin.wilkins", "key_options": null, "keyfile": "/home/jowilkin/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jowilkin", "validate_certs": true} 2021-11-11T20:38:53.614 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'bhubbard', 'ovpn': 'brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "bhubbard", "ovpn": "brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDP/yVttBEs7mpVfJL+reXb/BfB7DsLrA5ArVqEJjySi3NOz3e/wE8KRWbHD1UdraAU3P8b96GdAQxGOs4n0muOMejmaXWZCP3HkRL+rcd+03ZI1lb6ga9XjBARZfb2h+kg5jTJp/Ll6mwAYbSXvQ4MuDQzwGRDX5Q5Xme7OZ4Wb+vBmmkihHydySyP9YXbWWsBw4jvZCFSKMGsUVy/HIt1coZx/PyCG/K2Kl0nrE5ux2QqfdTIpzYo4c3y6uMIHDvt29lRmk3Gnh2//U5eN8mmRRKJlkvE8DKy3VYsl0JsP9pTWUZEYgV0EX251WTO0akQ2bKuszqd9Ai04RWnXUu5 brad@dhcp-1-165.bne.redhat.com", "key_options": null, "keyfile": "/home/bhubbard/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "bhubbard", "validate_certs": true} 2021-11-11T20:38:53.626 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'bhubbard', 'ovpn': 'brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "bhubbard", "ovpn": "brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDP/yVttBEs7mpVfJL+reXb/BfB7DsLrA5ArVqEJjySi3NOz3e/wE8KRWbHD1UdraAU3P8b96GdAQxGOs4n0muOMejmaXWZCP3HkRL+rcd+03ZI1lb6ga9XjBARZfb2h+kg5jTJp/Ll6mwAYbSXvQ4MuDQzwGRDX5Q5Xme7OZ4Wb+vBmmkihHydySyP9YXbWWsBw4jvZCFSKMGsUVy/HIt1coZx/PyCG/K2Kl0nrE5ux2QqfdTIpzYo4c3y6uMIHDvt29lRmk3Gnh2//U5eN8mmRRKJlkvE8DKy3VYsl0JsP9pTWUZEYgV0EX251WTO0akQ2bKuszqd9Ai04RWnXUu5 brad@dhcp-1-165.bne.redhat.com", "key_options": null, "keyfile": "/home/bhubbard/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "bhubbard", "validate_certs": true} 2021-11-11T20:38:53.844 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'yehudasa', 'ovpn': 'yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "yehudasa", "ovpn": "yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAyRDKb+InDsnW4E/S12Pr6rK3wDfUG3j+iw/NsCWwN5xpJEA1/NhVlRzY8/jejc6ddmnNKWbm/UcYVrbo7yClRTGu45g0Qm9lmfvgDgi1k8mwUCy+edbQ/R8FB9cIHHCnUuWzRmE2XGaeDiSVQgiYqq3mOPY/hwe3G7jwFalYPsbmY1jNqpzODmKFMRd1bOwdHEOMTfh0owHWqhhXakU8em30lYraQVYH24FplH2NUBArxmTuctyfweHxx7yH3ZFcXFUpUMSLuVDWb1A7fMiL6eW6wiwh+dD+q5iFNvuioZITP2EjIsb9nS+eM9bOJZuZ+5pln2EsHee27LjunbwgWw== yehudasa@fatty\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDF3PxJYgI3tS7SG+RnUvxMxr0/XwBAio0oiP4AC+o8rUYYyPd/RU27TR1kphbuJhQWVtaZ6kFSBQeNQUxv6rb8hmMzvlP1FDEoGkjaOHOuYEc2DFkkEAfaR2AlNKn3pUQgLE00TLHctCgd57Cn2Ppt8xEi/vTuo0gX3xjWOL4N9QDElCXKNdxdyyJ/RNb/Mid+YxgKKjf8ppmhYsQ+Qqu4YwPzLkfK1Qb3YtR6Hbjb4jxC7C1Z13h0vUgHe7qSmUCYNo9eC+lEgNmmnLb2sXJaUUJM8MEex6v+4muWM9hV0cq/Wplrx8fhMWMa978sxH8yVkE3Svqc/K4ZWN59l3xT yehudasa@yehudasa-XPS-L401X\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDvplQk8xkSXF03uSJh3H4zyZHRMjvgUJBgGN27Kb7 2021-11-11T20:38:53.846 INFO:teuthology.task.ansible.out:gAlIQv306wPKIprOwp+jDhEcPnEWBMRJMUJuBi0vEiqmDHt8mg8ZYcs+Gi7MbUgBokILdJuSG89KAgq7gBCuIhYbSg35pfvXvIZswiXvySqMrJNB495GYj/FrcWEBfQMuedMSkIfNzqPV9yR5YRh8fnSs3fjCzV96vzu7O4AzgwICbM6XmqVLXPax/32Hq0dz7sAawlQFCJYPHzVzjwmih46TRJnXrxewsRhk9GwKfI4SWaFIRdOdadGFSLgdZ+2iGE7hTZ3sRgkLOj+4zKa+xTbtjjXfw5up7J3hAmrK15Tz yehuda@yehuda-940X3G\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCfNxvAw6cb33jg36vw8kJu48/BttmFJ4BFeanYqwRgtWZrKpAAGHEgCiNeyIttMrI3t7Spcdyp6dy6n0P3WUFVgHWEYtGqXwXqDC9UXISkUJVVDZoEynpGC92RghhzH53+7jX1Ytpr30r/xb1ktryTPCoEJRqyrg2j1b167i6PV5tuJCA+77gW5LWBmMXJCprjnfnJ92Gp8ZewBB6gL1x+3RFkMxGsibL0Qw1CTKfyAHNYc/jos6YiL0b42Ha+NmKwvzZ1GgIleRMleisSOyKPUlPdR7UnmtEXa9bl3Gc/hc7Zt+v4iUypr2WL0X4Yl7rJNH/AAd/Z6jPkAX9nHYzh yehudasa@yehudasa-desktop\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCrSX8nDL2WaHe0b/rjE3TZhj5VwbrG6nn/hyWmm2dWcUqotSp973yD2G4tQL9UygeSkDGmjb6whrz2mcAFJ5P4oLtIcCLyZC550dcpwV3BtP/cMAIGE6xeEARKiP8KlVGFNDRg9/l1aAd08Bepe+rJyAYD7IxOOBfgKJMKbeOWnT40RTfwiD5CXNwjzQuhzZDEbm+8zjTChFM4P1EkfR3c+0X3MYpA1AsRQY0ElvFtcn6G0yPKR+obGdIs+ydGupKh1+peGWLemHMA5e62Bs4/Lwtyebyp7AOguC3ZBJ3TZWL4Vn1q6z21cQe6AuowjPrneZWC4ifFB6OUQJqM+R00pHBcajcvsfpYOblZvTJHM726Ek1vahOYfcbEMpqWlVYo+WPRUilzl5cI3ReyAcwphebqKPbhmzCuyJw3lWBKIVDhWjEFzGkxV/iFU5rIXFm/Z0JGNFFxmY1U4GlA4lFXtL7OvEdMsiZo1x0cNmFWLu0k9fQBtiQI19ZBiA7FPKc= yehuda@crestview", "key_options": null, "keyfile": "/home/yehudasa/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "yehudasa", "validate_certs": true} 2021-11-11T20:38:53.855 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'yehudasa', 'ovpn': 'yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "yehudasa", "ovpn": "yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAyRDKb+InDsnW4E/S12Pr6rK3wDfUG3j+iw/NsCWwN5xpJEA1/NhVlRzY8/jejc6ddmnNKWbm/UcYVrbo7yClRTGu45g0Qm9lmfvgDgi1k8mwUCy+edbQ/R8FB9cIHHCnUuWzRmE2XGaeDiSVQgiYqq3mOPY/hwe3G7jwFalYPsbmY1jNqpzODmKFMRd1bOwdHEOMTfh0owHWqhhXakU8em30lYraQVYH24FplH2NUBArxmTuctyfweHxx7yH3ZFcXFUpUMSLuVDWb1A7fMiL6eW6wiwh+dD+q5iFNvuioZITP2EjIsb9nS+eM9bOJZuZ+5pln2EsHee27LjunbwgWw== yehudasa@fatty\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDF3PxJYgI3tS7SG+RnUvxMxr0/XwBAio0oiP4AC+o8rUYYyPd/RU27TR1kphbuJhQWVtaZ6kFSBQeNQUxv6rb8hmMzvlP1FDEoGkjaOHOuYEc2DFkkEAfaR2AlNKn3pUQgLE00TLHctCgd57Cn2Ppt8xEi/vTuo0gX3xjWOL4N9QDElCXKNdxdyyJ/RNb/Mid+YxgKKjf8ppmhYsQ+Qqu4YwPzLkfK1Qb3YtR6Hbjb4jxC7C1Z13h0vUgHe7qSmUCYNo9eC+lEgNmmnLb2sXJaUUJM8MEex6v+4muWM9hV0cq/Wplrx8fhMWMa978sxH8yVkE3Svqc/K4ZWN59l3xT yehudasa@yehudasa-XPS-L401X\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDvplQk8xkSXF03uSJh3H4zyZHRMjvgUJBgGN27Kb7 2021-11-11T20:38:53.857 INFO:teuthology.task.ansible.out:gAlIQv306wPKIprOwp+jDhEcPnEWBMRJMUJuBi0vEiqmDHt8mg8ZYcs+Gi7MbUgBokILdJuSG89KAgq7gBCuIhYbSg35pfvXvIZswiXvySqMrJNB495GYj/FrcWEBfQMuedMSkIfNzqPV9yR5YRh8fnSs3fjCzV96vzu7O4AzgwICbM6XmqVLXPax/32Hq0dz7sAawlQFCJYPHzVzjwmih46TRJnXrxewsRhk9GwKfI4SWaFIRdOdadGFSLgdZ+2iGE7hTZ3sRgkLOj+4zKa+xTbtjjXfw5up7J3hAmrK15Tz yehuda@yehuda-940X3G\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCfNxvAw6cb33jg36vw8kJu48/BttmFJ4BFeanYqwRgtWZrKpAAGHEgCiNeyIttMrI3t7Spcdyp6dy6n0P3WUFVgHWEYtGqXwXqDC9UXISkUJVVDZoEynpGC92RghhzH53+7jX1Ytpr30r/xb1ktryTPCoEJRqyrg2j1b167i6PV5tuJCA+77gW5LWBmMXJCprjnfnJ92Gp8ZewBB6gL1x+3RFkMxGsibL0Qw1CTKfyAHNYc/jos6YiL0b42Ha+NmKwvzZ1GgIleRMleisSOyKPUlPdR7UnmtEXa9bl3Gc/hc7Zt+v4iUypr2WL0X4Yl7rJNH/AAd/Z6jPkAX9nHYzh yehudasa@yehudasa-desktop\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCrSX8nDL2WaHe0b/rjE3TZhj5VwbrG6nn/hyWmm2dWcUqotSp973yD2G4tQL9UygeSkDGmjb6whrz2mcAFJ5P4oLtIcCLyZC550dcpwV3BtP/cMAIGE6xeEARKiP8KlVGFNDRg9/l1aAd08Bepe+rJyAYD7IxOOBfgKJMKbeOWnT40RTfwiD5CXNwjzQuhzZDEbm+8zjTChFM4P1EkfR3c+0X3MYpA1AsRQY0ElvFtcn6G0yPKR+obGdIs+ydGupKh1+peGWLemHMA5e62Bs4/Lwtyebyp7AOguC3ZBJ3TZWL4Vn1q6z21cQe6AuowjPrneZWC4ifFB6OUQJqM+R00pHBcajcvsfpYOblZvTJHM726Ek1vahOYfcbEMpqWlVYo+WPRUilzl5cI3ReyAcwphebqKPbhmzCuyJw3lWBKIVDhWjEFzGkxV/iFU5rIXFm/Z0JGNFFxmY1U4GlA4lFXtL7OvEdMsiZo1x0cNmFWLu0k9fQBtiQI19ZBiA7FPKc= yehuda@crestview", "key_options": null, "keyfile": "/home/yehudasa/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "yehudasa", "validate_certs": true} 2021-11-11T20:38:54.063 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dang', 'ovpn': 'dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dang", "ovpn": "dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4"}, "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDx37xbP0jYqopygG66LbSMRPoqpGUPzQZ3G8eqJCk2I dang@revan.aa.redhat.com-2015-07-17", "key_options": null, "keyfile": "/home/dang/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dang", "validate_certs": true} 2021-11-11T20:38:54.084 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dang', 'ovpn': 'dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dang", "ovpn": "dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4"}, "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDx37xbP0jYqopygG66LbSMRPoqpGUPzQZ3G8eqJCk2I dang@revan.aa.redhat.com-2015-07-17", "key_options": null, "keyfile": "/home/dang/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dang", "validate_certs": true} 2021-11-11T20:38:54.292 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'branto', 'ovpn': 'branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "branto", "ovpn": "branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2YyDoeKdaCuJif79AxJAorqpUj33WHbHu/eIWpHju+iIF3f94K0qyEXgs4AwZin9pzfyd8N1aKNT8ZFPss6L0Xk68aO/6tdInq+1lTMMUC5v/LBqsIV2btPTFHrQWA0R10v05aZFvFaSqhHNsZaj98vQV7P+jiSs3yPWqCIWUTQ1VXnCeQnB4R5tob0gpMuonhkKq7vrJPhf6ge9Av+yjL7+8oP5LJqFJ5npDxkwZ0u3L0QQ9ftAT/e7Y7ESpwpgaI3Trl0EwAXFmiLjXEUA+FBbE7ko9kLqXqRjc1C0BdRG9sO0QKBGDBEx1W3w8rCPKgaDlVyqsBT1yvIjqqXDOQ== borix@dhcp-31-190.brq.redhat.com", "key_options": null, "keyfile": "/home/branto/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "branto", "validate_certs": true} 2021-11-11T20:38:54.313 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'branto', 'ovpn': 'branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "branto", "ovpn": "branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2YyDoeKdaCuJif79AxJAorqpUj33WHbHu/eIWpHju+iIF3f94K0qyEXgs4AwZin9pzfyd8N1aKNT8ZFPss6L0Xk68aO/6tdInq+1lTMMUC5v/LBqsIV2btPTFHrQWA0R10v05aZFvFaSqhHNsZaj98vQV7P+jiSs3yPWqCIWUTQ1VXnCeQnB4R5tob0gpMuonhkKq7vrJPhf6ge9Av+yjL7+8oP5LJqFJ5npDxkwZ0u3L0QQ9ftAT/e7Y7ESpwpgaI3Trl0EwAXFmiLjXEUA+FBbE7ko9kLqXqRjc1C0BdRG9sO0QKBGDBEx1W3w8rCPKgaDlVyqsBT1yvIjqqXDOQ== borix@dhcp-31-190.brq.redhat.com", "key_options": null, "keyfile": "/home/branto/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "branto", "validate_certs": true} 2021-11-11T20:38:54.522 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'xiaoxichen', 'ovpn': 'xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "xiaoxichen", "ovpn": "xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDO00mus2XPX/EfUZ/YsKI84TnWj5TbZP7Fs01EkoOTPgNoYysVdf0PwhAooz1qPrv+HUsUMpQT6uNYfCHyqx9rScR+XUlyQuFe7Sjp/dn3y+b1PKa6G4HX7f1eejR3ZaLZXA4R/8oucAXqQS5pDr3Y7prwkrp9cyk8uawBXoDcAlA1SMIPZPrtO1+iZw20NDhixsG7NE5jMFfw3q8xTckZEqBbteybci7X1v4lEYXCVlbSbUhyZ2Af8HEj09JBDOL2bI3yLvj9a5Oj/HtKubyApzZpA7rgeoii2UZQGuSRI1VJktp9YC0zGk5S843xYW92utbALKyQ6DdB2F5gc0DX xiaoxchen@LM-SHC-16501223", "key_options": null, "keyfile": "/home/xiaoxichen/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "xiaoxichen", "validate_certs": true} 2021-11-11T20:38:54.543 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'xiaoxichen', 'ovpn': 'xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "xiaoxichen", "ovpn": "xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDO00mus2XPX/EfUZ/YsKI84TnWj5TbZP7Fs01EkoOTPgNoYysVdf0PwhAooz1qPrv+HUsUMpQT6uNYfCHyqx9rScR+XUlyQuFe7Sjp/dn3y+b1PKa6G4HX7f1eejR3ZaLZXA4R/8oucAXqQS5pDr3Y7prwkrp9cyk8uawBXoDcAlA1SMIPZPrtO1+iZw20NDhixsG7NE5jMFfw3q8xTckZEqBbteybci7X1v4lEYXCVlbSbUhyZ2Af8HEj09JBDOL2bI3yLvj9a5Oj/HtKubyApzZpA7rgeoii2UZQGuSRI1VJktp9YC0zGk5S843xYW92utbALKyQ6DdB2F5gc0DX xiaoxchen@LM-SHC-16501223", "key_options": null, "keyfile": "/home/xiaoxichen/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "xiaoxichen", "validate_certs": true} 2021-11-11T20:38:54.750 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ffilz', 'ovpn': 'ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ffilz", "ovpn": "ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCz7e2G3Ziaus2yMviKh9s65e+yaqy1+ezs5IC6US90/8WumNAWqC4Yr3ckvCc120r3VwpJWcvaAWCF+4IZeO43lSNLadbCnJONed5YklcUuIJvjC1eDbZTMLJ7h0ijgNXgAFmFq0AV2NHuE2L2VKXk2hsIqDVGjZLUpi7W+ic07xnScEoXXDSk5mhJ6bKmFsUYCC91g2roYs9Ui5fGd+ey28SFOADI7osDLpLkiTF2OzGel93o5qobNZwKR0AxGsBIta17mvf3I3307bGnvugDXpNB0FaS1QZhIzMO4G4LZL9zahCvv4wm5GmuO4F0+Ac7dwFu3KZKtHQe3g6WHuAv ffilz@redhat.com", "key_options": null, "keyfile": "/home/ffilz/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ffilz", "validate_certs": true} 2021-11-11T20:38:54.772 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ffilz', 'ovpn': 'ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ffilz", "ovpn": "ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCz7e2G3Ziaus2yMviKh9s65e+yaqy1+ezs5IC6US90/8WumNAWqC4Yr3ckvCc120r3VwpJWcvaAWCF+4IZeO43lSNLadbCnJONed5YklcUuIJvjC1eDbZTMLJ7h0ijgNXgAFmFq0AV2NHuE2L2VKXk2hsIqDVGjZLUpi7W+ic07xnScEoXXDSk5mhJ6bKmFsUYCC91g2roYs9Ui5fGd+ey28SFOADI7osDLpLkiTF2OzGel93o5qobNZwKR0AxGsBIta17mvf3I3307bGnvugDXpNB0FaS1QZhIzMO4G4LZL9zahCvv4wm5GmuO4F0+Ac7dwFu3KZKtHQe3g6WHuAv ffilz@redhat.com", "key_options": null, "keyfile": "/home/ffilz/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ffilz", "validate_certs": true} 2021-11-11T20:38:54.989 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'joao', 'ovpn': 'joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "joao", "ovpn": "joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCpGZK2cWUtE85Bt7fl+kSeiC0lM/EN+H2X3ePGq3LiD4g5R5n6nJsLaJaIZXdJ96G0JSnj07G8Xu1CkVbGDHuVg2O9tnxLhz6tzDwFYNFrQIpALlQwOq8OiwGDDyFkMXQ3JqCCWvyD0HLysOJQIBIlb/o92ZCSAGOyNX1JjrhXF+mlBk1AUW11UeZOeGZQ34MKHapH7T/DuexCz5gfZI/jY9Rg3hr4jb1zGttvAF2eeLJCjqD/Vi4mowYB/Kn/PJhNixeA2sli2i8r6yO/bFmA/PWnfSl+EqD4mUJWb5ATeJ3hyW8qVLOogY6XVu4Sd1VgL7l1RYzXuxJV3Rb1Yw2T joao@sepia\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDV0jbKpbmXW1bhRIOBIdVV1L73qjls7s5pxiW2EMPvZLYzxLBE92e3Vf8bSMWlm1Lyb7a+u5BR85+2c+h2F59sgU1Saek5ebPpFU5qBn8AzYH18iMVSPciByOzwS0NXwWnksckWxUkQgP9bZP/1ORQ7jWZEOHZ5mEQ0oMkP0bfG1cpFrNTxOhDmwbCiQ8dp6KeZaMg4hErALr5Eh/EESxKIkuwQzk9bDbtjPRDu0F2dE0NCjs0k0BXmDA80GksxuOacRxE95K6/er11qst/VqA7o6BfhxJMTXWMdj8NhzIUNjZ8LqLTG6P/XGR24sA2d2WBhlEKRMZNqeEjzmi1REr joao@tagide", "key_options": null, "keyfile": "/home/joao/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "joao", "validate_certs": true} 2021-11-11T20:38:55.011 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'joao', 'ovpn': 'joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "joao", "ovpn": "joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCpGZK2cWUtE85Bt7fl+kSeiC0lM/EN+H2X3ePGq3LiD4g5R5n6nJsLaJaIZXdJ96G0JSnj07G8Xu1CkVbGDHuVg2O9tnxLhz6tzDwFYNFrQIpALlQwOq8OiwGDDyFkMXQ3JqCCWvyD0HLysOJQIBIlb/o92ZCSAGOyNX1JjrhXF+mlBk1AUW11UeZOeGZQ34MKHapH7T/DuexCz5gfZI/jY9Rg3hr4jb1zGttvAF2eeLJCjqD/Vi4mowYB/Kn/PJhNixeA2sli2i8r6yO/bFmA/PWnfSl+EqD4mUJWb5ATeJ3hyW8qVLOogY6XVu4Sd1VgL7l1RYzXuxJV3Rb1Yw2T joao@sepia\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDV0jbKpbmXW1bhRIOBIdVV1L73qjls7s5pxiW2EMPvZLYzxLBE92e3Vf8bSMWlm1Lyb7a+u5BR85+2c+h2F59sgU1Saek5ebPpFU5qBn8AzYH18iMVSPciByOzwS0NXwWnksckWxUkQgP9bZP/1ORQ7jWZEOHZ5mEQ0oMkP0bfG1cpFrNTxOhDmwbCiQ8dp6KeZaMg4hErALr5Eh/EESxKIkuwQzk9bDbtjPRDu0F2dE0NCjs0k0BXmDA80GksxuOacRxE95K6/er11qst/VqA7o6BfhxJMTXWMdj8NhzIUNjZ8LqLTG6P/XGR24sA2d2WBhlEKRMZNqeEjzmi1REr joao@tagide", "key_options": null, "keyfile": "/home/joao/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "joao", "validate_certs": true} 2021-11-11T20:38:55.219 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'nhm', 'ovpn': 'nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "nhm", "ovpn": "nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHhYb/mGgHbEntAN2iPFTV1k8MYDs+jtsJDPE9MWtUZXksdeG8cEfXDBGhiBsprjELsW4+czJBqqESugFMevEysTVKIo5lc95bf1nApxfiRyMnAxb5wQ2OeLY62x26TWJhPu6HMajV+gb8CKjxDobf1Ix4ELbYlL77SOH1VVDtVZ6HiwtYngXAPMONCdI99cuwJlBpv5RjDoLwljOoxEXIh+1Fsvth+iymp0rIPuJx91DzYYgWZFfCLPHRgDy0H7YRdQWUzhrxTyzuT27vsNh5L8HgdowzOu9f2HwGzlTlxli94IJLMBY2359xsqjSswZzAv+BiPDAZMUwJpYvRvSt nhm@clusterfaq.org\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1JAKymiRDyVar5P75cMycxz9u8VsLuNtf/0t1na4ofa4YmanKswPQQz1Am82pvyPqXlBbdIGVneyMrkEoFAk4nTtfmCl7WdtaxnH87Dr9BOLEp+lIPHy7jpS/dcVmKRPB1qCYZ5GNAZ9g+DrXdaeOJdUOpIK49BP1IK8Uw5eBWAx/ae6/12yz+d/6e9kBvO7OUamMSne6Ghe2pRoI8jFOdWMbQy3ofb5wSHZGPNDn0YTGbNWHlfEAcrFmGQSTwohca4SqCJ1NgKnYu4qPVmb3Gbpm+MpUPs8Gy++rQ8wVZ5xWZW/yReIbJCqqVY4vrQzVywQXRucIrB1VyvShuNW3 nhm@metropolis\nssh-dss AAAAB3NzaC1kc3MAAACBAM7BfQAinQmLlkTgt7IIg3nQyyaJRiWA4agfu/HncrcAmepGqPCwy451g1eLMD3AC4QptHnV80Y47YQTyX1VQixIQ6M9zHG6Bo7IXzyqFHFpqnDZN25DaMM4na5bf1CwC9RtR7VWubojatPylOpflkChUBuAGWnzwkz2cxl4Dc47AAAAFQD35l/Lg4eTqZ83eTgJ5lihzHS/awAAAIB/ZdMSE6M0zEvdNPsWY28bvQ0QXN7w1ncVXGachA1JOEFGYvsbHeoywLpSYoiPLV57/wZT2dUzG70u80IQGildZ1WgQNn7C9r2Z3ZUgKmRGyC+PpYEa6nv+U4CHdMViBRXxNrP0bj2CfQv 2021-11-11T20:38:55.221 INFO:teuthology.task.ansible.out:UxUxrHWtOu/da2W23cpSs6Wz8gtz2QAAAIEAvf4vx/32Sy+kbvqOCYz0wu0ItlECN0qOk3gAMvphYMI2sW2ez2+Tsua4eoPRXT10zFxiRTkHQd5FSIvNWXV2iqed6rpwTu5K8vJz2duMIyI+xWLHo4XudtZdNxggkf+RW9JvPOMLMp4sCzBXxpreHBsb1CynndKeYVrgMLybZ28= mark@Marks-MacBook-Air.local\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCfDOM2v4h6Dk4pMWPQxxHI5rhFtLjkme/4hAp6ki/kJ1XOSzBQdQmD2UPDKvfO9lKO7v12G2Wt+re+0giAcSkfej4+xqV53u7d2pTbMI5U3zohZt8AgESx22DAxj1wBZYuyGO2TRI7DRQMxC47E8EJgcmZBBnMKk1Rz8k71pTmg+CI2vjWx/2x2bup9Mqe4uMA8ubjg2qRlWEqPKIKcHZyrAPdIlpy2FSZ9SPa0mJpdl6aYcFNgonD8BO6A0u2RPO90KmEWrRWu69vRvbbTDiqE0CkJJlAlu3jeFVD8US9vymBHSI0BDFUbukuyG1tAujytqGBRLzxQDU9JOYMP45v nhm@mocha", "key_options": null, "keyfile": "/home/nhm/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "nhm", "validate_certs": true} 2021-11-11T20:38:55.241 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'nhm', 'ovpn': 'nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "nhm", "ovpn": "nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHhYb/mGgHbEntAN2iPFTV1k8MYDs+jtsJDPE9MWtUZXksdeG8cEfXDBGhiBsprjELsW4+czJBqqESugFMevEysTVKIo5lc95bf1nApxfiRyMnAxb5wQ2OeLY62x26TWJhPu6HMajV+gb8CKjxDobf1Ix4ELbYlL77SOH1VVDtVZ6HiwtYngXAPMONCdI99cuwJlBpv5RjDoLwljOoxEXIh+1Fsvth+iymp0rIPuJx91DzYYgWZFfCLPHRgDy0H7YRdQWUzhrxTyzuT27vsNh5L8HgdowzOu9f2HwGzlTlxli94IJLMBY2359xsqjSswZzAv+BiPDAZMUwJpYvRvSt nhm@clusterfaq.org\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1JAKymiRDyVar5P75cMycxz9u8VsLuNtf/0t1na4ofa4YmanKswPQQz1Am82pvyPqXlBbdIGVneyMrkEoFAk4nTtfmCl7WdtaxnH87Dr9BOLEp+lIPHy7jpS/dcVmKRPB1qCYZ5GNAZ9g+DrXdaeOJdUOpIK49BP1IK8Uw5eBWAx/ae6/12yz+d/6e9kBvO7OUamMSne6Ghe2pRoI8jFOdWMbQy3ofb5wSHZGPNDn0YTGbNWHlfEAcrFmGQSTwohca4SqCJ1NgKnYu4qPVmb3Gbpm+MpUPs8Gy++rQ8wVZ5xWZW/yReIbJCqqVY4vrQzVywQXRucIrB1VyvShuNW3 nhm@metropolis\nssh-dss AAAAB3NzaC1kc3MAAACBAM7BfQAinQmLlkTgt7IIg3nQyyaJRiWA4agfu/HncrcAmepGqPCwy451g1eLMD3AC4QptHnV80Y47YQTyX1VQixIQ6M9zHG6Bo7IXzyqFHFpqnDZN25DaMM4na5bf1CwC9RtR7VWubojatPylOpflkChUBuAGWnzwkz2cxl4Dc47AAAAFQD35l/Lg4eTqZ83eTgJ5lihzHS/awAAAIB/ZdMSE6M0zEvdNPsWY28bvQ0QXN7w1ncVXGachA1JOEFGYvsbHeoywLpSYoiPLV57/wZT2dUzG70u80IQGildZ1WgQNn7C9r2Z3ZUgKmRGyC+PpYEa6nv+U4CHdMViBRXxNrP0bj2CfQv 2021-11-11T20:38:55.242 INFO:teuthology.task.ansible.out:UxUxrHWtOu/da2W23cpSs6Wz8gtz2QAAAIEAvf4vx/32Sy+kbvqOCYz0wu0ItlECN0qOk3gAMvphYMI2sW2ez2+Tsua4eoPRXT10zFxiRTkHQd5FSIvNWXV2iqed6rpwTu5K8vJz2duMIyI+xWLHo4XudtZdNxggkf+RW9JvPOMLMp4sCzBXxpreHBsb1CynndKeYVrgMLybZ28= mark@Marks-MacBook-Air.local\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCfDOM2v4h6Dk4pMWPQxxHI5rhFtLjkme/4hAp6ki/kJ1XOSzBQdQmD2UPDKvfO9lKO7v12G2Wt+re+0giAcSkfej4+xqV53u7d2pTbMI5U3zohZt8AgESx22DAxj1wBZYuyGO2TRI7DRQMxC47E8EJgcmZBBnMKk1Rz8k71pTmg+CI2vjWx/2x2bup9Mqe4uMA8ubjg2qRlWEqPKIKcHZyrAPdIlpy2FSZ9SPa0mJpdl6aYcFNgonD8BO6A0u2RPO90KmEWrRWu69vRvbbTDiqE0CkJJlAlu3jeFVD8US9vymBHSI0BDFUbukuyG1tAujytqGBRLzxQDU9JOYMP45v nhm@mocha", "key_options": null, "keyfile": "/home/nhm/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "nhm", "validate_certs": true} 2021-11-11T20:38:55.439 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mkidd', 'ovpn': 'linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mkidd", "ovpn": "linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJD+2BHStkaMJ8RStIIz+yDJCGNT1wbwUnmuO71vNlFxzIyHGvCq3ktx2xTwZz/kj6ZayXYWIRH//BhK2/iK/v+u2WQMBURfMHbf0lDMmNfbHCDBjYvEYgFo4Jwvzq4WHus0AHh9TdVt+TpcxyGPhVbUJh8NZut+w4cRIKu2xGoZZGDeLgv9ijXvDPrCdBxtpvQnF99C0bCTdPM7MK1FSCQEaRmNUgzEj0H7R+gmWi0T4HUD8ZY7y0vWbYPqT3Z07My1u2BfRBYpXnv1655/b777TkSlzSasORhSTT0AkW25u90Rsj2JWr39+49/Ia9uOyFtZIirugEv2v7v/l0grN linuxkidd@zenbook", "key_options": null, "keyfile": "/home/mkidd/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mkidd", "validate_certs": true} 2021-11-11T20:38:55.471 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mkidd', 'ovpn': 'linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mkidd", "ovpn": "linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJD+2BHStkaMJ8RStIIz+yDJCGNT1wbwUnmuO71vNlFxzIyHGvCq3ktx2xTwZz/kj6ZayXYWIRH//BhK2/iK/v+u2WQMBURfMHbf0lDMmNfbHCDBjYvEYgFo4Jwvzq4WHus0AHh9TdVt+TpcxyGPhVbUJh8NZut+w4cRIKu2xGoZZGDeLgv9ijXvDPrCdBxtpvQnF99C0bCTdPM7MK1FSCQEaRmNUgzEj0H7R+gmWi0T4HUD8ZY7y0vWbYPqT3Z07My1u2BfRBYpXnv1655/b777TkSlzSasORhSTT0AkW25u90Rsj2JWr39+49/Ia9uOyFtZIirugEv2v7v/l0grN linuxkidd@zenbook", "key_options": null, "keyfile": "/home/mkidd/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mkidd", "validate_certs": true} 2021-11-11T20:38:55.668 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jlopez', 'ovpn': 'JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jlopez", "ovpn": "JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDrRvNeBa+/GXxkSDmaim7U1wFpdBvgcK9/eP1I4fPS5xAWyxXI0s0ryX4Jwb7auhZdcqPjgUPQvIVVRridKJPz7l6jUlbIqdnZu4VHuaGG/DfIVxsSmw5aI5qI9Fg/s16JPwOUuzcLHsSSh8tFxY9L1yvs3ztQnKHW6nN5+zhPbW8UjmTNUksXXAeCM/OZn+IOOzzvfh0vVe/mghIW2JpOjHkZv4qwaWYcHg6+wWE4m8l84VIegDBiONknZXcQaRj82nHtfdnb2FSiV4+gKP18i90ai0iTKKXUkMJNFKp1T5MuF7UuJLA7l10FmgQJsZBS2Loa5oeqbHYeztNAfUxF JCLO@oh-mbp-jcl", "key_options": null, "keyfile": "/home/jlopez/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jlopez", "validate_certs": true} 2021-11-11T20:38:55.700 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jlopez', 'ovpn': 'JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jlopez", "ovpn": "JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDrRvNeBa+/GXxkSDmaim7U1wFpdBvgcK9/eP1I4fPS5xAWyxXI0s0ryX4Jwb7auhZdcqPjgUPQvIVVRridKJPz7l6jUlbIqdnZu4VHuaGG/DfIVxsSmw5aI5qI9Fg/s16JPwOUuzcLHsSSh8tFxY9L1yvs3ztQnKHW6nN5+zhPbW8UjmTNUksXXAeCM/OZn+IOOzzvfh0vVe/mghIW2JpOjHkZv4qwaWYcHg6+wWE4m8l84VIegDBiONknZXcQaRj82nHtfdnb2FSiV4+gKP18i90ai0iTKKXUkMJNFKp1T5MuF7UuJLA7l10FmgQJsZBS2Loa5oeqbHYeztNAfUxF JCLO@oh-mbp-jcl", "key_options": null, "keyfile": "/home/jlopez/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jlopez", "validate_certs": true} 2021-11-11T20:38:55.908 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'haomaiwang', 'ovpn': 'yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "haomaiwang", "ovpn": "yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDFkUz8Wr2RPiY8+xf2hL4qMT+jA4mo1gmpIF8q7uVR03vX5NhzQVxVnXiyLCFv6susSMxpBafa7aLutqigXhDGJWUbsJII5yTnFYVer0QmudEsLmoDyvI1HCNk1IVY6aIESE50N3Ndx+agWasIU8TGKqJdDMPyiUtfrvsrGWi/91ZLX6aJSrFqBJtTpxDoNe4XL6QRPNv0wb8GF0az5graXkzVe3GkaQS9lyw8m2ougDvW1SLvoeE+lhUUGhJGkkCZKv1f+QNLhUHyQ3XkXqh4MrP3pPUKIuQ7nKiWDjCP6Db+ZcejjTRC19D2gLjhrw3Z2J7nn/loijlFzV5dp+BL haomaiwang@gmail.com", "key_options": null, "keyfile": "/home/haomaiwang/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "haomaiwang", "validate_certs": true} 2021-11-11T20:38:55.929 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'haomaiwang', 'ovpn': 'yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "haomaiwang", "ovpn": "yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDFkUz8Wr2RPiY8+xf2hL4qMT+jA4mo1gmpIF8q7uVR03vX5NhzQVxVnXiyLCFv6susSMxpBafa7aLutqigXhDGJWUbsJII5yTnFYVer0QmudEsLmoDyvI1HCNk1IVY6aIESE50N3Ndx+agWasIU8TGKqJdDMPyiUtfrvsrGWi/91ZLX6aJSrFqBJtTpxDoNe4XL6QRPNv0wb8GF0az5graXkzVe3GkaQS9lyw8m2ougDvW1SLvoeE+lhUUGhJGkkCZKv1f+QNLhUHyQ3XkXqh4MrP3pPUKIuQ7nKiWDjCP6Db+ZcejjTRC19D2gLjhrw3Z2J7nn/loijlFzV5dp+BL haomaiwang@gmail.com", "key_options": null, "keyfile": "/home/haomaiwang/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "haomaiwang", "validate_certs": true} 2021-11-11T20:38:56.137 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kchai', 'ovpn': 'kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kchai", "ovpn": "kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAuTvkaUtEaLbLcYA9Zw3MoHI5ZPudKf6CAbAZdhp4RMVujimR61TT6LdRF20g6F7pH7GrI/1WP2bu7a3sZyCZ1nCJCOhrTTS+2wwy7oGo/0yY4l+Bw6i7LGmJgqpO5dDxwADMlaHUwJ7Rf1Q+Pd7NmggkAVTmaN6qUGffz9hZsj7ibD/Z8/GFtQsCEMloTLtnz4NB6Gxe54UXEKz9pvjpXBhs0xWsJtDKaPFrbKRKswuH45XPuwJhGr249ZYtwQqcgXSWhOUcPTwf5Y6fj27aL2dx566DxQzqzNbcTcO96MguIy8Ufnc0kIOQQw9oHJZtg82zwM6+2mWr9J/K94Gv9w== kefu@save.node2", "key_options": null, "keyfile": "/home/kchai/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kchai", "validate_certs": true} 2021-11-11T20:38:56.159 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kchai', 'ovpn': 'kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kchai", "ovpn": "kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAuTvkaUtEaLbLcYA9Zw3MoHI5ZPudKf6CAbAZdhp4RMVujimR61TT6LdRF20g6F7pH7GrI/1WP2bu7a3sZyCZ1nCJCOhrTTS+2wwy7oGo/0yY4l+Bw6i7LGmJgqpO5dDxwADMlaHUwJ7Rf1Q+Pd7NmggkAVTmaN6qUGffz9hZsj7ibD/Z8/GFtQsCEMloTLtnz4NB6Gxe54UXEKz9pvjpXBhs0xWsJtDKaPFrbKRKswuH45XPuwJhGr249ZYtwQqcgXSWhOUcPTwf5Y6fj27aL2dx566DxQzqzNbcTcO96MguIy8Ufnc0kIOQQw9oHJZtg82zwM6+2mWr9J/K94Gv9w== kefu@save.node2", "key_options": null, "keyfile": "/home/kchai/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kchai", "validate_certs": true} 2021-11-11T20:38:56.366 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vumrao", "ovpn": "vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC56FiAgzz9aF+qAMkw5aGoCyoSA8Lf2LPaz3QxTYEzmQt0ByXMmVw0cgUI4gDt7hMBX6LoxPALZ7C1aH8aPTpb6+hxHGfir2tqQmU49JOQiUvBP/Xtt1ilxMou9Wu/ZadEhIZxaP/TBncXNls9cbFs9bV7Q8J3CLCTGRa8kilEpZ7209c/xQI4yQ6q+Q3H4FkO0YHP+Tkn9mg1df+BKkXfSBycjAyOXPvljoetjw0FHEbJ7lTCcSG+hCb2LEACo2OK9eWC3GYwDPKZF1Ht9HfDvg4bHfY3OVOixYiZSim7dbj3+VHWWsiH64usGH1o6fn4vV16UhVJiiicPMWvllzH vikhyat.umrao@gmail.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC9G/I5iaw5d4ssyJ4iTUxSvIBAX+P03VVHsNMHAB70rVGGwBnBJFqJNbukzxfnsqwzN6vSeuMy7I1zc2VVi4GtBQPK0iCbVQN+Njq0VCIxb4hLmjD3Vn8NgRp80955VJyYGT9+/JbZY1UCdPBuld6hsnTipkdo5U3huAHD4i1FpYKol+4DCZHAO5wNilCbests0+ZPtHYe53vnNDz685cx/f7RTtEarvU2fECxkurlBBNvuIl6SN8bo+NaJ01Li0xAD+2fpGz6Iu0WUEdWTkt78ubvA4If2k8XAKxkOKEvfyX+AtTzdHxWSQu7ixkbuksuK6A0bnankn+DdiP6Y293 vumrao@redhat", "key_options": null, "keyfile": "/home/vumrao/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vumrao", "validate_certs": true} 2021-11-11T20:38:56.387 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vumrao", "ovpn": "vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC56FiAgzz9aF+qAMkw5aGoCyoSA8Lf2LPaz3QxTYEzmQt0ByXMmVw0cgUI4gDt7hMBX6LoxPALZ7C1aH8aPTpb6+hxHGfir2tqQmU49JOQiUvBP/Xtt1ilxMou9Wu/ZadEhIZxaP/TBncXNls9cbFs9bV7Q8J3CLCTGRa8kilEpZ7209c/xQI4yQ6q+Q3H4FkO0YHP+Tkn9mg1df+BKkXfSBycjAyOXPvljoetjw0FHEbJ7lTCcSG+hCb2LEACo2OK9eWC3GYwDPKZF1Ht9HfDvg4bHfY3OVOixYiZSim7dbj3+VHWWsiH64usGH1o6fn4vV16UhVJiiicPMWvllzH vikhyat.umrao@gmail.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC9G/I5iaw5d4ssyJ4iTUxSvIBAX+P03VVHsNMHAB70rVGGwBnBJFqJNbukzxfnsqwzN6vSeuMy7I1zc2VVi4GtBQPK0iCbVQN+Njq0VCIxb4hLmjD3Vn8NgRp80955VJyYGT9+/JbZY1UCdPBuld6hsnTipkdo5U3huAHD4i1FpYKol+4DCZHAO5wNilCbests0+ZPtHYe53vnNDz685cx/f7RTtEarvU2fECxkurlBBNvuIl6SN8bo+NaJ01Li0xAD+2fpGz6Iu0WUEdWTkt78ubvA4If2k8XAKxkOKEvfyX+AtTzdHxWSQu7ixkbuksuK6A0bnankn+DdiP6Y293 vumrao@redhat", "key_options": null, "keyfile": "/home/vumrao/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vumrao", "validate_certs": true} 2021-11-11T20:38:56.599 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dfuller', 'ovpn': 'dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dfuller", "ovpn": "dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC/r/Xf58szqJ3wLdTE+mRK1Z11f/+VABmx6804NO77ncdC5ZvD3Kx6WP/oje01iwWEVXNAgEOaXp6j+U10aPvBMaS30qf/IhUcrXwO1YcGzZ0pV6UZMxcTuJw6w1VXy/YtgW5BlIfSm78zabztES9BqCgjYYM2rml3L1teDF4WU1QiI97iW0R57yc0t/gqXGpxqSFQ+iNYAl0Q1MGUgYltLvPrhm9WI2WhR2/GivdVMEM5lViNlegxneAZRb9YZMCG2kmSP+9Zd2NR367OrlgzmgeY/9u+KGXQiyfaO330s+iksxVwwE3fDl+KnqXuHSJNccJV7iXzZKej2QAire4X dfuller@Douglass-MacBook-Pro.local", "key_options": null, "keyfile": "/home/dfuller/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dfuller", "validate_certs": true} 2021-11-11T20:38:56.610 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dfuller', 'ovpn': 'dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dfuller", "ovpn": "dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC/r/Xf58szqJ3wLdTE+mRK1Z11f/+VABmx6804NO77ncdC5ZvD3Kx6WP/oje01iwWEVXNAgEOaXp6j+U10aPvBMaS30qf/IhUcrXwO1YcGzZ0pV6UZMxcTuJw6w1VXy/YtgW5BlIfSm78zabztES9BqCgjYYM2rml3L1teDF4WU1QiI97iW0R57yc0t/gqXGpxqSFQ+iNYAl0Q1MGUgYltLvPrhm9WI2WhR2/GivdVMEM5lViNlegxneAZRb9YZMCG2kmSP+9Zd2NR367OrlgzmgeY/9u+KGXQiyfaO330s+iksxVwwE3fDl+KnqXuHSJNccJV7iXzZKej2QAire4X dfuller@Douglass-MacBook-Pro.local", "key_options": null, "keyfile": "/home/dfuller/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dfuller", "validate_certs": true} 2021-11-11T20:38:56.828 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'owasserm', 'ovpn': 'owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "owasserm", "ovpn": "owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDIKepJjIovBTCkdVyvAdptljhw9bYcwtIqV/ojdbpeuJtxDo9UyrEkNoLeVpZgOcrbgrkOE7AEixo3qB9PbD8z5NRUleV7wipSh3QwrDmSylSVReJKOwYdxUKkzesZYBGIGYzAbTKqcVzPUWCeW314+xmv90XkO/+1vgbC5uoXfbfnaiDHQimBom8QV5Egr+EhhcuCqMEqaVSvjMr1T0e6IL9gUens6Mys6czI37go6QvmgEzkBbDJJ64PsiOOFHVNk4DJVJ1c7TRtsnpIHM1YVFjsH1AnAL6UufbgNSFf5dziUB/Bz815P1/jsOGpHkUKE37DbHU5xtLGl3MmimMt4ndcG6EggBGEYBwfEWPRAsZs3qngssrNH5W0CzbysLaoovzq5VqS8SdLMu5XrRLBEB0uWWzLq2bobYymk3t/oby5zrHJV78xAuLmldqjvlPZSdkgwWNv9YBLkteBVr2HhjtuashJ6amNPbA4kCGhXMpeRhfvfqs31CBEKbTV+A8ODWbWAoQxLCcqS24x0yMi73N0UzuqDTaJMcE8ODJvOxcLNaV688QDbrC/UehpX/Bp/DM7fLHj4YhkRpZxkkN5w2kb/bBbWya4DrffhHKqn9CH1z5ynj0237PhTwCXjtRn0JzpeRgN8OraFgR2H3uPc9XNqlkFxxOBK0lRngbxzw== owasserm@redhat.com", "key_options": null, "keyfile": "/home/owasserm/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "owasserm", "validate_certs": true} 2021-11-11T20:38:56.850 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'owasserm', 'ovpn': 'owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "owasserm", "ovpn": "owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDIKepJjIovBTCkdVyvAdptljhw9bYcwtIqV/ojdbpeuJtxDo9UyrEkNoLeVpZgOcrbgrkOE7AEixo3qB9PbD8z5NRUleV7wipSh3QwrDmSylSVReJKOwYdxUKkzesZYBGIGYzAbTKqcVzPUWCeW314+xmv90XkO/+1vgbC5uoXfbfnaiDHQimBom8QV5Egr+EhhcuCqMEqaVSvjMr1T0e6IL9gUens6Mys6czI37go6QvmgEzkBbDJJ64PsiOOFHVNk4DJVJ1c7TRtsnpIHM1YVFjsH1AnAL6UufbgNSFf5dziUB/Bz815P1/jsOGpHkUKE37DbHU5xtLGl3MmimMt4ndcG6EggBGEYBwfEWPRAsZs3qngssrNH5W0CzbysLaoovzq5VqS8SdLMu5XrRLBEB0uWWzLq2bobYymk3t/oby5zrHJV78xAuLmldqjvlPZSdkgwWNv9YBLkteBVr2HhjtuashJ6amNPbA4kCGhXMpeRhfvfqs31CBEKbTV+A8ODWbWAoQxLCcqS24x0yMi73N0UzuqDTaJMcE8ODJvOxcLNaV688QDbrC/UehpX/Bp/DM7fLHj4YhkRpZxkkN5w2kb/bBbWya4DrffhHKqn9CH1z5ynj0237PhTwCXjtRn0JzpeRgN8OraFgR2H3uPc9XNqlkFxxOBK0lRngbxzw== owasserm@redhat.com", "key_options": null, "keyfile": "/home/owasserm/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "owasserm", "validate_certs": true} 2021-11-11T20:38:57.057 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vasu', 'ovpn': 'vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vasu", "ovpn": "vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVFy1sjGvvHwS7Q6E61ykH4dUGfxW9+a7Xo3JMpTBT33/E8bTrYF6ZHSdH+uxScisV9Vll222NJCLXyx2s9EOcv4L88vWDTZGCaUb3iA5dweLm7EYi1NXT/p+jWDA7MAF1FQGSAQafTHpNRxLcrVPgsLDy35weV4xj7ShEU+w7EBDb75jlJXqy0luJLKNBrLzz/wDV9Oe6y3APwQBhtO7vGrVzrWEF/LWLe7U9ad3DQjvLM7+H+QVpNy+nt7us73K8vHDk1KrPlCCjrcN+fSGPD/ILlxrDrDeqmmBCtPMa7PSSjCyAV/EW0fsmArcHrHXKeKWTKXG9sDL0KBqos1kB vasu@vasuSrv\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQConeaJ2kvUOMy+jeKvXZTw655L3KK0ZG0HONIySFrRNkGa/M/hYf8oMNVh9SEax7vLNme0kwk7d71EhgtMGNsvHtQOkshWxM9P0XPk4HgB/OL+2LQ/7M3SwD4vuXPbqvrQS63YMoxcHBue5Ol5hXmM+qSoZajpUrfB2woqbp7aKsCpoK4Cgj4XfLHfh0n239ae38DU6GoAcL0GzMsxMdQhWzBwOO+ZHfowohc1pfY0+xGIMW1OlMN1eO7kuK2gGS/YRVOnLw9uu/2NErIjBzCihNN0PYWur2JE+KmU1x6VMaky49cKmAlc5WUqZ+FpV3gvnb3cLjBClnCkdjIcLeRR vasukulkarni@ovpn-116-227.phx2.redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKRFyOO7qitrq6Hk2NSl28h4laRTxA2zezy1wwjTqs6UL+oZjNRYQjeWD3dDNPZrUmc9JyI3j1EN11cCsKAp8YXz3ItB46M0BPeJ4TXCpg36vxs7/twSDMq/EoXA5sRBbUJFf+bthcNemwG1WsJoxRVzMezbWBTHvz5i4Hbh4sL3PBbac3FhT/3b5Q4tKdO6bXGXmNLAoALyVZZMfoSz9CWzvrO+g8Houb9iJiaXHp293y+ND9Wicl9wHPckO1VcocZ+ldByujgX1cz6m+pxmf6AKY1 2021-11-11T20:38:57.059 INFO:teuthology.task.ansible.out:OMJj5PjmpN0HrgpV2QLVA+ba9qBiDwi1g8sRHXkOETMZ2jq/2a/mmTtvrsF vasu@teuthology", "key_options": null, "keyfile": "/home/vasu/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vasu", "validate_certs": true} 2021-11-11T20:38:57.079 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vasu', 'ovpn': 'vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vasu", "ovpn": "vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVFy1sjGvvHwS7Q6E61ykH4dUGfxW9+a7Xo3JMpTBT33/E8bTrYF6ZHSdH+uxScisV9Vll222NJCLXyx2s9EOcv4L88vWDTZGCaUb3iA5dweLm7EYi1NXT/p+jWDA7MAF1FQGSAQafTHpNRxLcrVPgsLDy35weV4xj7ShEU+w7EBDb75jlJXqy0luJLKNBrLzz/wDV9Oe6y3APwQBhtO7vGrVzrWEF/LWLe7U9ad3DQjvLM7+H+QVpNy+nt7us73K8vHDk1KrPlCCjrcN+fSGPD/ILlxrDrDeqmmBCtPMa7PSSjCyAV/EW0fsmArcHrHXKeKWTKXG9sDL0KBqos1kB vasu@vasuSrv\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQConeaJ2kvUOMy+jeKvXZTw655L3KK0ZG0HONIySFrRNkGa/M/hYf8oMNVh9SEax7vLNme0kwk7d71EhgtMGNsvHtQOkshWxM9P0XPk4HgB/OL+2LQ/7M3SwD4vuXPbqvrQS63YMoxcHBue5Ol5hXmM+qSoZajpUrfB2woqbp7aKsCpoK4Cgj4XfLHfh0n239ae38DU6GoAcL0GzMsxMdQhWzBwOO+ZHfowohc1pfY0+xGIMW1OlMN1eO7kuK2gGS/YRVOnLw9uu/2NErIjBzCihNN0PYWur2JE+KmU1x6VMaky49cKmAlc5WUqZ+FpV3gvnb3cLjBClnCkdjIcLeRR vasukulkarni@ovpn-116-227.phx2.redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKRFyOO7qitrq6Hk2NSl28h4laRTxA2zezy1wwjTqs6UL+oZjNRYQjeWD3dDNPZrUmc9JyI3j1EN11cCsKAp8YXz3ItB46M0BPeJ4TXCpg36vxs7/twSDMq/EoXA5sRBbUJFf+bthcNemwG1WsJoxRVzMezbWBTHvz5i4Hbh4sL3PBbac3FhT/3b5Q4tKdO6bXGXmNLAoALyVZZMfoSz9CWzvrO+g8Houb9iJiaXHp293y+ND9Wicl9wHPckO1VcocZ+ldByujgX1cz6m+pxmf6AKY1 2021-11-11T20:38:57.080 INFO:teuthology.task.ansible.out:OMJj5PjmpN0HrgpV2QLVA+ba9qBiDwi1g8sRHXkOETMZ2jq/2a/mmTtvrsF vasu@teuthology", "key_options": null, "keyfile": "/home/vasu/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vasu", "validate_certs": true} 2021-11-11T20:38:57.286 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'smohan', 'ovpn': 'shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "smohan", "ovpn": "shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDlOImwCV0AGu5L29ABaefV9Vqjv43WYJYFuLTxGQQdeQGcways6iNKGIX/nzf3UWkwgg25OVSpzB7zf6FjBx+HcQU+RJ7FLzeaz0s8ZxBXqWwtOYOvP9cXmAXA07hD5UAfxZKC1twBzL82bKJ2ZSmzfQmBjvpUOqRs7XwsQIClOW/ycY+jLgLRLFFDxzADsgnn6UEGlHeMETYocUbe89P3wbY5fHhqZmn6e5ncuAQMsiLRa9d+QhSdpD07Y/P65EzqRF4UOX+zpOc6gPlgHFir9pvf0bTbbBG/qKN9+KGqzZMRJGKenpHM6EMfLkf0QmrT4s4zd4ZnUQPq3HN3pul3 redhat@unused\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD9n4ege98BNmvgUxogdcpptMILFfEisAQaiLUvSKpTsoy2gF7C6xSJXgDaziT5dW4ad2El4xmKQ/e6/e14Vx5i9pGbJz4NvL36Xmb3ent2PtY18i6d3CbnmGI8J7YH4YS9QvR7FC/KuvwLR6AdJ+uQShGhqG8kWJMjbqrtaxu8l1hq8ymBXw3O1Ht+xrZ/r3dEkAIVF3bMCSUnZpEbSsNoeiydEQl0ZCw+meXYmSw15vYVtKpYscZxqADqhSJJxIEHZA4E88dq6aYdV2AZ/s4cJK+tBl/ckBnclI3VGHSGn6oaHoO/fCHtKiENplDuKoLGQPO6xJ2Q/apHSGD7McY3 redhat@network\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDn/XgNfvqytLHgfBUdQIhJiWBlj+JrXqGZYvAZZJIWs32Ugm1/lCQTssGZrEj3+Qv81jLfkP6VQ8gXSWJkzUK4csxRxkZKXqcYLDE1csdmAFDOfWUFJwhMsCiiCyBiW9oapioLpvd40nw8k9qM4QuzaUhVonoGXMDDL1/dCgkbIjiQxxTJPHDEJFjtzul+VuhlhHqE6S1syoJG+UKo3Q1tOBCs8rwaPXEfCXFsyv+ZmcEv1+mpM8QRjdr1D1hfUfwb4CHtaa2AE6B5wL+zoY6+jsrTwZSwCfeIdgEZ8e3V2iXP5F7G6eXotu5r7mGYJClHZFLSfbNPN5J+EbRORl9/ wbhack3r@dhcp35-24.lab.eng.blr.redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHYkp1nuRSYSAl6me1xuoTU30DW2YpAR6EH1c2adUC9tNtTZbMnab11aitQLAtZb9SKsRgnoB5KpH+zJ0VNKCuJU227r9vZwcVUruSGwA6X7LOrSkK0gFqNRZkVoKhC9oGHSU0zegYBDxx8VLxSV3CE56b+KEosEo6iJe7TkXy3wXHK6am0uhQPEdSHtdUdD683CGIffdF1RFmAFnzTPn96lC9rLe1mIGO+jVQx4zIo3i33RFQLy++4DtQw6Lrgp26Xr85HmQekjgyiya0VkR9Ypy7fgF2ucBGegnOgZDGCCY4G2RuqiZyK 2021-11-11T20:38:57.288 INFO:teuthology.task.ansible.out:z2TZaOecDwrLkbYM7EGp8up5uLFRZsV shmohan@shmohan.remote.csb", "key_options": null, "keyfile": "/home/smohan/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "smohan", "validate_certs": true} 2021-11-11T20:38:57.308 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'smohan', 'ovpn': 'shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "smohan", "ovpn": "shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDlOImwCV0AGu5L29ABaefV9Vqjv43WYJYFuLTxGQQdeQGcways6iNKGIX/nzf3UWkwgg25OVSpzB7zf6FjBx+HcQU+RJ7FLzeaz0s8ZxBXqWwtOYOvP9cXmAXA07hD5UAfxZKC1twBzL82bKJ2ZSmzfQmBjvpUOqRs7XwsQIClOW/ycY+jLgLRLFFDxzADsgnn6UEGlHeMETYocUbe89P3wbY5fHhqZmn6e5ncuAQMsiLRa9d+QhSdpD07Y/P65EzqRF4UOX+zpOc6gPlgHFir9pvf0bTbbBG/qKN9+KGqzZMRJGKenpHM6EMfLkf0QmrT4s4zd4ZnUQPq3HN3pul3 redhat@unused\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD9n4ege98BNmvgUxogdcpptMILFfEisAQaiLUvSKpTsoy2gF7C6xSJXgDaziT5dW4ad2El4xmKQ/e6/e14Vx5i9pGbJz4NvL36Xmb3ent2PtY18i6d3CbnmGI8J7YH4YS9QvR7FC/KuvwLR6AdJ+uQShGhqG8kWJMjbqrtaxu8l1hq8ymBXw3O1Ht+xrZ/r3dEkAIVF3bMCSUnZpEbSsNoeiydEQl0ZCw+meXYmSw15vYVtKpYscZxqADqhSJJxIEHZA4E88dq6aYdV2AZ/s4cJK+tBl/ckBnclI3VGHSGn6oaHoO/fCHtKiENplDuKoLGQPO6xJ2Q/apHSGD7McY3 redhat@network\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDn/XgNfvqytLHgfBUdQIhJiWBlj+JrXqGZYvAZZJIWs32Ugm1/lCQTssGZrEj3+Qv81jLfkP6VQ8gXSWJkzUK4csxRxkZKXqcYLDE1csdmAFDOfWUFJwhMsCiiCyBiW9oapioLpvd40nw8k9qM4QuzaUhVonoGXMDDL1/dCgkbIjiQxxTJPHDEJFjtzul+VuhlhHqE6S1syoJG+UKo3Q1tOBCs8rwaPXEfCXFsyv+ZmcEv1+mpM8QRjdr1D1hfUfwb4CHtaa2AE6B5wL+zoY6+jsrTwZSwCfeIdgEZ8e3V2iXP5F7G6eXotu5r7mGYJClHZFLSfbNPN5J+EbRORl9/ wbhack3r@dhcp35-24.lab.eng.blr.redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHYkp1nuRSYSAl6me1xuoTU30DW2YpAR6EH1c2adUC9tNtTZbMnab11aitQLAtZb9SKsRgnoB5KpH+zJ0VNKCuJU227r9vZwcVUruSGwA6X7LOrSkK0gFqNRZkVoKhC9oGHSU0zegYBDxx8VLxSV3CE56b+KEosEo6iJe7TkXy3wXHK6am0uhQPEdSHtdUdD683CGIffdF1RFmAFnzTPn96lC9rLe1mIGO+jVQx4zIo3i33RFQLy++4DtQw6Lrgp26Xr85HmQekjgyiya0VkR9Ypy7fgF2ucBGegnOgZDGCCY4G2RuqiZyK 2021-11-11T20:38:57.309 INFO:teuthology.task.ansible.out:z2TZaOecDwrLkbYM7EGp8up5uLFRZsV shmohan@shmohan.remote.csb", "key_options": null, "keyfile": "/home/smohan/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "smohan", "validate_certs": true} 2021-11-11T20:38:57.516 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'abhi', 'ovpn': 'abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "abhi", "ovpn": "abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDZePxJmYerihzA0aQbMt7K7bctPEDcwPHiO8EVuJowp6zwhSmtDnYuFybKJsee4l8+rDKR6T5Ugdl5pLjSzQ+Ba00OSH7+qWC6shvVDYju5mIooJnc+kwx8RgapbGtDdl/yYskBb1KD5g/3+i/2aM+Igk+7nQR7Cny5+WnUEvHd7PHBbUXQSuu+mlUlnXCVMM8g8qlc1I3WQL5eQoIqGjSWRgZUIsn3P5oryYBzbuG9jqLwbaohJvZRJqkimQDe7zCFk6bB3z39pyOWUN+sldRvyL8FPyEmqog4IwpjcYyCuCnPhnb6fNq9yB65RUIde82Kkzx3LZrYOXTrdHgD/lr abhi@trusty", "key_options": null, "keyfile": "/home/abhi/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "abhi", "validate_certs": true} 2021-11-11T20:38:57.538 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'abhi', 'ovpn': 'abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "abhi", "ovpn": "abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDZePxJmYerihzA0aQbMt7K7bctPEDcwPHiO8EVuJowp6zwhSmtDnYuFybKJsee4l8+rDKR6T5Ugdl5pLjSzQ+Ba00OSH7+qWC6shvVDYju5mIooJnc+kwx8RgapbGtDdl/yYskBb1KD5g/3+i/2aM+Igk+7nQR7Cny5+WnUEvHd7PHBbUXQSuu+mlUlnXCVMM8g8qlc1I3WQL5eQoIqGjSWRgZUIsn3P5oryYBzbuG9jqLwbaohJvZRJqkimQDe7zCFk6bB3z39pyOWUN+sldRvyL8FPyEmqog4IwpjcYyCuCnPhnb6fNq9yB65RUIde82Kkzx3LZrYOXTrdHgD/lr abhi@trusty", "key_options": null, "keyfile": "/home/abhi/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "abhi", "validate_certs": true} 2021-11-11T20:38:57.755 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'cbodley', 'ovpn': 'cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "cbodley", "ovpn": "cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDcsBe6CzgWh64iZAH8zbU6ra4yEiUBSsiUn0AHzyQurvjeRpL2Mut6B1WEi0kJG7BAM6b13JrhnD9BW9yIMIUIPpVTa2o4zaXP90+bvjGrgDeJbxv3T9/TGh13k+WWyampxm2lo90/+Paql/HObQyyIv7Wan0DOdIe2/Ue4ooWH0tu3kj+3unJTuw82oEDEn+9tGs0oleG2c8XN0gQyVl1DzR2Gqf9PCzXK7hInC3niY2huoSAZzVFEgsniKEWqTnVWYi49X2SsJhUyHCj+mqrTG3YnW30nha3EXOWIbQGblXVoTIOc+gNreqdGByOMICKSnQmPSG4DAl3c1Y4h4A71T3Q043IUv7v7S9tAEKq2h6oJ9I/a5z0u4kaL/+B7Cw7l4bitTdcZ0rcqSvOkKUNNgxujbucMSSc4Tsh5L0762xaziD9nQZl5bteb3VtsMAueqhu7L3sJy69+HGg7oF68t5TguErXfI2UNVGOn3eRj+zN/zSwLUXMoROlpM5yHXSzHvpsbgZiHBImM2CeG4qaPtqzFw7VH6NWJv4ypb0htOMWsbXmLshNTca1KXrrBcVozSJC8wKgEOxu4spk26G8x1IVTsi2ypyYX+irtAqZrrpIPdQ0Ss9NjKPhFNXAxJgLHj6TlH8UNh1gRfN/QTyzpFSe4EH6uOlnWuxcuM8ZQ== cbodley@redhat.com", "key_options": null, "keyfile": "/home/cbodley/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "cbodley", "validate_certs": true} 2021-11-11T20:38:57.777 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'cbodley', 'ovpn': 'cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "cbodley", "ovpn": "cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDcsBe6CzgWh64iZAH8zbU6ra4yEiUBSsiUn0AHzyQurvjeRpL2Mut6B1WEi0kJG7BAM6b13JrhnD9BW9yIMIUIPpVTa2o4zaXP90+bvjGrgDeJbxv3T9/TGh13k+WWyampxm2lo90/+Paql/HObQyyIv7Wan0DOdIe2/Ue4ooWH0tu3kj+3unJTuw82oEDEn+9tGs0oleG2c8XN0gQyVl1DzR2Gqf9PCzXK7hInC3niY2huoSAZzVFEgsniKEWqTnVWYi49X2SsJhUyHCj+mqrTG3YnW30nha3EXOWIbQGblXVoTIOc+gNreqdGByOMICKSnQmPSG4DAl3c1Y4h4A71T3Q043IUv7v7S9tAEKq2h6oJ9I/a5z0u4kaL/+B7Cw7l4bitTdcZ0rcqSvOkKUNNgxujbucMSSc4Tsh5L0762xaziD9nQZl5bteb3VtsMAueqhu7L3sJy69+HGg7oF68t5TguErXfI2UNVGOn3eRj+zN/zSwLUXMoROlpM5yHXSzHvpsbgZiHBImM2CeG4qaPtqzFw7VH6NWJv4ypb0htOMWsbXmLshNTca1KXrrBcVozSJC8wKgEOxu4spk26G8x1IVTsi2ypyYX+irtAqZrrpIPdQ0Ss9NjKPhFNXAxJgLHj6TlH8UNh1gRfN/QTyzpFSe4EH6uOlnWuxcuM8ZQ== cbodley@redhat.com", "key_options": null, "keyfile": "/home/cbodley/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "cbodley", "validate_certs": true} 2021-11-11T20:38:57.975 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mwatts', 'ovpn': 'mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mwatts", "ovpn": "mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDrtHyRuOSEXAM6mwL4qoM+NM4hpIKhaRS+/Y6RDoFKszkXFw/jC9ZYh7q39rQpWQBKHVe4RyPN9yyGWydFwaR/BjZi14ippmYOqVG4f7b+k39NFbMJRcSxYC+NvkqxK7+zpsnIWC+RdJ+blZi+zCbaaAYcMDSvBZW19FsJSLrO7hh4R1cuzgV+Xrmwdc1/VH+HqPMu3tHGv7N0SfsklWI8Inr0aiL83QqqgKPcVxljSWZGYvvFyTQOrkNQeaLURJNER9WIIac9o57WOINw8kR+ojebcOUBsrxZH9j+gMjsJ752Rph03wdMUpFcnS6yfvI4Ial7x/sGms4FE+wL/vVF mdw@pika", "key_options": null, "keyfile": "/home/mwatts/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mwatts", "validate_certs": true} 2021-11-11T20:38:58.007 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mwatts', 'ovpn': 'mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mwatts", "ovpn": "mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDrtHyRuOSEXAM6mwL4qoM+NM4hpIKhaRS+/Y6RDoFKszkXFw/jC9ZYh7q39rQpWQBKHVe4RyPN9yyGWydFwaR/BjZi14ippmYOqVG4f7b+k39NFbMJRcSxYC+NvkqxK7+zpsnIWC+RdJ+blZi+zCbaaAYcMDSvBZW19FsJSLrO7hh4R1cuzgV+Xrmwdc1/VH+HqPMu3tHGv7N0SfsklWI8Inr0aiL83QqqgKPcVxljSWZGYvvFyTQOrkNQeaLURJNER9WIIac9o57WOINw8kR+ojebcOUBsrxZH9j+gMjsJ752Rph03wdMUpFcnS6yfvI4Ial7x/sGms4FE+wL/vVF mdw@pika", "key_options": null, "keyfile": "/home/mwatts/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mwatts", "validate_certs": true} 2021-11-11T20:38:58.226 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "prsrivas", "ovpn": "pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC0zCoXv2Kkxjqv5FOlJZ2PRA9tStdXAVGZJbSLOhAWeqHQL6O2l8MO5GFzHz4FA/g7lbNzLKaCbG1wYGuL9FOt0o36/eHykgENu00SlFunIQn+T/WNaGxrZqnfbdS9TdImFAnVY4vuzrBhFTb6qcqTLhw1WVz+l/iOv5CoY+7NnILWcj8dHNcPzU1ihlFAbvQgQcMtt1130zqc6L8o+SDm2YQF2nuOWxj0LbD6YpRn3AqWsTN5QbqzdeaJiUt6kdywisPdoiSWLuivRKhNLNhnWO+B4DnchwfApy8ovJShHZAsrSoc+a+RQDmOiLXmc5wAvNnREHbqod8/0bQ9vSXaVQCt1gnnK8Bf7GQElg3Z3aep4D4w6rBB+LBsHFbTfrzWAHkXxJeGiQTlJGz0IYHi1weWrzY6x5rEA0gYDu0FVLOLrsYDtuTQpjcTwpmaHCsxz6wU3wCNmTsV/AZhniFHJhHXacCmDp1JfsI2kP9FjcJJE0SygnZ/SzKe0DHamnyT1OgdQaG/KX2eRv/vXUPg2ukKiA1fyFnFeEsWK6rUO0YgCswqjrM4vU0leZ1b8vK9aM4IDJM3TJqdMLfOa0rPntekNOqXUn213CP+Yiz9l1TH4zX3P/Kv6Vv1gTeGcX0lMaYhowng+97j3LLqGcSUPMklrKen/xrsACFxN9fxaQ== prsrivas@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCks+XmOepYEdSBP6U0T+r+fZowSS7UHY5W8AUtAkvXiKlTG518luG2wtSALIsB5+jyjC44ITwV9X/vvsDaXQnp1hFanwWeuU1jDdy+MvywvxI9q4naWlDpxrfq4i/AioSvdoIo02uUFYPKTCHrm7wBjTwpWbCD0gbC9/OGAHM88Md3F0qbgFEr3BERvNGe0coCjWRMS8+GTZXS22WW1/NxNzpdCUjO/9j4QydpW7UF1kqVhnPVZozmh4wl0rk8tgb9Zx3vCUBG1Q5eO/q+HsW54eXLa9FcuQcoJnNdV2hMXRHHZHTgaPKZ/ildukzZHo0i/Dq1mi5N3s5A7aDYxOWWlRXazwL8zu/UQVrxVRni8vDTFk5682lskF1WifzADaqicUWzasdK0PC4G7JmHPi+XHskeOt5i4ywEnFRXnYtMOIu0CNO82Rg4CdUAD58YXCGoQ4r9TG4LlwKg75ouNHB/ToJidSg9NBwclP+MICcTpWItxVXpBj/YkeLcntHtVTFNH 2021-11-11T20:38:58.228 INFO:teuthology.task.ansible.out:ORxIwIwWng94wVEDCfyXelyAR6VfwcPlWWrbzhbEO9DH40GuluQi38ySWi/HpnrVriJMLzEQA9OPrLsME1LDgPGkF2GloPL8nnnLWB9L+YEBNNAVoSo0PHBsjIibdZ/SZm7zUgYGp4fuG+m1+SDOjcfD6GSXOTQwQou+knEQ== prsrivas@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDISEXvGI2oJLTRR15ePZLkmdYjaQJxt9V/yBwKdGqYvnPzmLOpdmVpdQJo7ZOWwdp5tJsCy4q5uX5vIUv6Gyi3uVSGmZnrKUbsqoDNwAX5vx/gFBINIs/ZXXyD4fpUT8aoZeq1XmXdIaWG0nImsrtRnRxKP9NGZEcW8z0WPwgF0Ck+RH0qZUix6Bfbgy7vMXnivuDv+NdjEwyszcrPWvf6Krr0am9Q0+9nj/mjvk6CZLl+CH3HNdkhZuNs+qMG4XW8KhLvQB2p790XVlTHuVU4jWrFmjymggZxkhoCY/ukHH6wi2Pbx6yjjKOj8iIw1xKabJzJ7S8UKT23A41d918DUCUuEG66pN6Z4dnv+fZJxj5iEo1GdycSWIc4SpLTK3z4zDZypVKIzjI4o8VwQaN48HkVjUdTICTvGQfytFi3jX1I5nxIZiiPcxMQLR5hQggP8pn2XtvXpomn0aBc8tVuuZJLVq4zCVnhlVB6IACmb71evjFGOsoSuEsaTufUr48= prsrivas@t480.lab.blr.redhat.com", "key_options": null, "keyfile": "/home/prsrivas/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "prsrivas", "validate_certs": true} 2021-11-11T20:38:58.238 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "prsrivas", "ovpn": "pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC0zCoXv2Kkxjqv5FOlJZ2PRA9tStdXAVGZJbSLOhAWeqHQL6O2l8MO5GFzHz4FA/g7lbNzLKaCbG1wYGuL9FOt0o36/eHykgENu00SlFunIQn+T/WNaGxrZqnfbdS9TdImFAnVY4vuzrBhFTb6qcqTLhw1WVz+l/iOv5CoY+7NnILWcj8dHNcPzU1ihlFAbvQgQcMtt1130zqc6L8o+SDm2YQF2nuOWxj0LbD6YpRn3AqWsTN5QbqzdeaJiUt6kdywisPdoiSWLuivRKhNLNhnWO+B4DnchwfApy8ovJShHZAsrSoc+a+RQDmOiLXmc5wAvNnREHbqod8/0bQ9vSXaVQCt1gnnK8Bf7GQElg3Z3aep4D4w6rBB+LBsHFbTfrzWAHkXxJeGiQTlJGz0IYHi1weWrzY6x5rEA0gYDu0FVLOLrsYDtuTQpjcTwpmaHCsxz6wU3wCNmTsV/AZhniFHJhHXacCmDp1JfsI2kP9FjcJJE0SygnZ/SzKe0DHamnyT1OgdQaG/KX2eRv/vXUPg2ukKiA1fyFnFeEsWK6rUO0YgCswqjrM4vU0leZ1b8vK9aM4IDJM3TJqdMLfOa0rPntekNOqXUn213CP+Yiz9l1TH4zX3P/Kv6Vv1gTeGcX0lMaYhowng+97j3LLqGcSUPMklrKen/xrsACFxN9fxaQ== prsrivas@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCks+XmOepYEdSBP6U0T+r+fZowSS7UHY5W8AUtAkvXiKlTG518luG2wtSALIsB5+jyjC44ITwV9X/vvsDaXQnp1hFanwWeuU1jDdy+MvywvxI9q4naWlDpxrfq4i/AioSvdoIo02uUFYPKTCHrm7wBjTwpWbCD0gbC9/OGAHM88Md3F0qbgFEr3BERvNGe0coCjWRMS8+GTZXS22WW1/NxNzpdCUjO/9j4QydpW7UF1kqVhnPVZozmh4wl0rk8tgb9Zx3vCUBG1Q5eO/q+HsW54eXLa9FcuQcoJnNdV2hMXRHHZHTgaPKZ/ildukzZHo0i/Dq1mi5N3s5A7aDYxOWWlRXazwL8zu/UQVrxVRni8vDTFk5682lskF1WifzADaqicUWzasdK0PC4G7JmHPi+XHskeOt5i4ywEnFRXnYtMOIu0CNO82Rg4CdUAD58YXCGoQ4r9TG4LlwKg75ouNHB/ToJidSg9NBwclP+MICcTpWItxVXpBj/YkeLcntHtVTFNH 2021-11-11T20:38:58.239 INFO:teuthology.task.ansible.out:ORxIwIwWng94wVEDCfyXelyAR6VfwcPlWWrbzhbEO9DH40GuluQi38ySWi/HpnrVriJMLzEQA9OPrLsME1LDgPGkF2GloPL8nnnLWB9L+YEBNNAVoSo0PHBsjIibdZ/SZm7zUgYGp4fuG+m1+SDOjcfD6GSXOTQwQou+knEQ== prsrivas@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDISEXvGI2oJLTRR15ePZLkmdYjaQJxt9V/yBwKdGqYvnPzmLOpdmVpdQJo7ZOWwdp5tJsCy4q5uX5vIUv6Gyi3uVSGmZnrKUbsqoDNwAX5vx/gFBINIs/ZXXyD4fpUT8aoZeq1XmXdIaWG0nImsrtRnRxKP9NGZEcW8z0WPwgF0Ck+RH0qZUix6Bfbgy7vMXnivuDv+NdjEwyszcrPWvf6Krr0am9Q0+9nj/mjvk6CZLl+CH3HNdkhZuNs+qMG4XW8KhLvQB2p790XVlTHuVU4jWrFmjymggZxkhoCY/ukHH6wi2Pbx6yjjKOj8iIw1xKabJzJ7S8UKT23A41d918DUCUuEG66pN6Z4dnv+fZJxj5iEo1GdycSWIc4SpLTK3z4zDZypVKIzjI4o8VwQaN48HkVjUdTICTvGQfytFi3jX1I5nxIZiiPcxMQLR5hQggP8pn2XtvXpomn0aBc8tVuuZJLVq4zCVnhlVB6IACmb71evjFGOsoSuEsaTufUr48= prsrivas@t480.lab.blr.redhat.com", "key_options": null, "keyfile": "/home/prsrivas/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "prsrivas", "validate_certs": true} 2021-11-11T20:38:58.456 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'pdonnell', 'ovpn': 'pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pdonnell", "ovpn": "pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQD3vdP9QHyNFUNPOdplezEbdYsQfcC91ncATI6Fl/xHifmUbcvjIRqZHkCY6FNWUpMeg9S3+wlHJtzjVqJJ/MlQML3EtwnzQkeWv8bjtuZ07OqULYIv0zEx63XUfSnrdFleJ8+KDSpn7krkYE44q85B88IL+ZBeq5eCXFJ1TtmsDcvsN8WqZNWMdUGqgH3Bb4D8+YKrEfGxoievyxZ1A1DnBHan2l8foTKYggRrnuSbTb59cFpxY3R7Arc5Cn9kDJT1nDVBA5SSGCfRMiNyBOUzQVYe/UjKTaCpngglXC3wYnjoGvGcpksHLQCxyuJPB6djC1tdF+3Jehdf8y8+2TBzWXiiPq2XwS58sER5GcLLLQv26pPL8hgu6CdI0sHxgcgQBgpjoLwUMHmv13CnN/AAcZY4RRDK2X5zd6wFSEhuuCoUqh/m7K219IxI6fVkLaApC851D/ua/pj9OMjiOaJ+D13cPrgfqxva4eK8FQZvqQm7aUK/TT216QNYmn7hkzNjMTkjv/L5yYQ5Ym92AHy44wHzLPrtDB5mk94vHOjG+KlAt9ofuLncMSAXWe+fOqxuIQ6VfbL2LUv0EGYhHe4XOVj98q5utsVQxQbXAZzNQaq9FWXs2A4xcVP7aWj0Fooi17LN/W0ng4Iijir5hyLr7k7M1G6resgiY7p83biNow== pdonnell@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQCpgLVt7Zphmf6nqD+9BC0N91jdmaY6ZpvrYWlzbhxXITQ6juIEtegnlZ+8ohdeEfHFBiyu1xOSQSM1X1ugvpPQE94UqQg7JVb0tuoUCNJHYnIuv8T4YgyP0/vEV48k2G0BBWlP+IJ5f1iDvZuLHWLR5JSgGqh5ELaaHr7Cbyb6JlImldb7SDIoPHMCZ+n6dLO8l5Zy7qWelw6aeA770xmPucBr6t/BVns2g77XgJz6zFd99yEaDy27i8jfzL7WqR9i93yVE9EsL7tuOjg6G9KUzxBF0bg9iVsJXypvcBuG0JwQ+SDKqYw/cN4iLy5qBa6Siqrfpc/LKU21VNmJ0e5eXNgKKbMBpBb8GVmkoLaSRcXw7LclWopjNDA+meUhDAwU7b97N7cqN8BYhBKQyiShXFHMKLuNo2fSuTRt/icfb9VisKNGBdvwgV0oOlrqbXMzZKmj9amb9b+3JDBTGB7ioCsj5x/Evf9srXRrQ9kikIr1aBbVXIWir/lasNWVydcmUYsNi8L7W0vXArLwp+GgOXJFeZUiD3z9LG4CouRxMSwoj3lgWeL2ZR8feH4D/SP/+QmBdU6+aG97au9qreeXi5X8QTR3LND0DwwEKczX+3s+Fa950Oieom/gXC57PVKVh90KVoAGw3HzJTlZrauOtLkYbTUAX6woOMhAtJ5oWuDDT4UrHQj37jRrrl2dETLwKwUJN2+2L07vnnOZ4ABcGWLsU6d3bXeUBlw3pdsYqPebDKUEpjL2kUOw+zmfjE 2021-11-11T20:38:58.458 INFO:teuthology.task.ansible.out:fdVx9BfrvExvDX+3MkxccCJ8FY7od1s5fGEFHsXnIiMnLO/HZb/JN+Aoe+/ExadIcYwJnWcCJyly2/532kGFUlzOliCwPxgJpjWCKQAv6LjWmxoPXiA4JI0bp6y7lMEjmV2Bp3mwphlsW/r8Ie5hg1UPh/ttHRp/VZgcV8e6nNmP/qc6k9b2DWTNFqtWqm1hXLOwq08HzUEeH9mTqvenyAbv2RFMOS18Y2K+4BKUzHwZujrz9qEh/kqtpFScEIXkOJ+HZz3mI6XfjWFIPS487yRGfUvW3+nSWhvHc8vJY21bFRKdjD84qCVlZ2BfOCGUZSKftVItEiqXMTRs3RZd5epfCVZbJuA0hG16jUseeHqqZqDtlbLkzkxtS6v3iGRnPh/Ke/94FHHCzvYY8cmMYAVaM6p/G3edWftbaEtN70Fo7VKxPrpPzuSRZ7Vxmnb+Zad95DxxAC/vtMy/vkBVIP6dyn4HGxqa1oECFMe4oXdhnxZQoPzvWxb8upcXeAeOMMbcIlMq5a+dCBPGFNKhNhrb+9uAEDHwjE7rZlMJVSvt9v1S4hll6jiYIap+7n4reSKrS45aO9U0xiKXJJDohh pdonnell@icewind\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQC8MdUJzIvTCn0fM+hv1HAO/Lwjy/C/V7stQRWGIxyrRHbniyX8iuaU3j7YNFglSQPJq+YRM9UnGauM9/0J9tHKLsMp+zRQYrtxU/ThiTUN8TRJY11mGUol4yLsqkTtm6oq5sl6o+LfuA4hzaYc3XLrFHRJuCUgbtPG71Q0C/y3U+itrDK4wXUxBa4YzZzcIMg+c88VXwoouPbiL0gW5PCXzMaNtKlOqBUKlE935MLn70wIEfN/EwAAbss0ga1moqeqmi9Nlvm20DNIVFmkAGtYoq6CHXtL/uehzai5Sueop8Vmd8+1U8gDTo6JpiagdD8ZNf0zAbRx587KAT5ZMnahJpO276j7RNa85ZFkNkEnBLqBHRkCkTCYq1GehYyRaR69uuKyczFiIoTfAQgPh7VlnpT9525b/4nFsBDOkWxxWAIJR61YiKemPV/n/e9SwhvVsMDb2B/RBriwR7Mi2SZ/lBHGB+z0OA+EE2rGZy4LFVOfUczqIdpltlNotXan4M4F361i+bX19KE+f2RLySn3+GiFv0CdLwslS/y4xysz+FWxw6B0Y/EkpiDU3CdWM61+THGGm/DVhfL/Pvca7k8PonHFYw/x5w1nBcADpkM9JF+m5eOnp+weTEzl58sS+3GnoAuL8YFJwMW1RmE50xM0o5E3SDc8XftUvaSZz8mhII3uwDDukbELfmkLbnIeipQKyjO+GEx0JBp4Itz82qGuao0Wi32U9wHzhSWnX4GQ0pKZnq9frSlyCNwHu/iMiKF8sA2bX3dvplFJ9nKWPefcU8bdl/upr5hYrd2GttjUSkWZa+UuF264yx3ztcN7jotRhEmmdEcHVzKQ26/NyRI4/9DQpDKY7zw4O2wFkXWg5L9kodXwWzgy4cyOr4dMD9LBVLOpHc0CAji5/+lfuYQbu1mwVdfOzCXDqByjt7zCbeN7KuUh403sJ8PTQVp642xA9i4AkPv5g+fqi5C0++1DbZP0lmLfKtTswHhoN+kAqZu2N4Mu6mNVNKyZgW5h1pZzNSGWYcZttyYmMCHhPketReL0LPJ+VUsqQCVohuta3aaSe44InA28GZsXzMj5B0el/84bnE7cN4b+rfGD25NT73JEt3RgPa9W9jqT1hwd/nf53BDKpp7DlHjR77WSWT4h4F7bhbM1RsGSqAfZjCUJT6Rq+IuM2XcrLDY8zIeArkqaZLXMzfGj6KJ7cQes05P5LYfY4osXiq68Gs/7gzF31PQRRGOYAoqMN7752sn0ZMcpnUj3mJutQcncRUN4+naTMjU36WWD7it0YuQBnb2IJ4iKjYLInw8pr2zKgWtgr1TSCM4xKp/v1/ZMmbBmS+d 2021-11-11T20:38:58.459 INFO:teuthology.task.ansible.out:TzNC7Mpkt8vdCNjzZ5KZ5 pdonnell@aglarond", "key_options": null, "keyfile": "/home/pdonnell/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pdonnell", "validate_certs": true} 2021-11-11T20:38:58.477 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'pdonnell', 'ovpn': 'pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pdonnell", "ovpn": "pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQD3vdP9QHyNFUNPOdplezEbdYsQfcC91ncATI6Fl/xHifmUbcvjIRqZHkCY6FNWUpMeg9S3+wlHJtzjVqJJ/MlQML3EtwnzQkeWv8bjtuZ07OqULYIv0zEx63XUfSnrdFleJ8+KDSpn7krkYE44q85B88IL+ZBeq5eCXFJ1TtmsDcvsN8WqZNWMdUGqgH3Bb4D8+YKrEfGxoievyxZ1A1DnBHan2l8foTKYggRrnuSbTb59cFpxY3R7Arc5Cn9kDJT1nDVBA5SSGCfRMiNyBOUzQVYe/UjKTaCpngglXC3wYnjoGvGcpksHLQCxyuJPB6djC1tdF+3Jehdf8y8+2TBzWXiiPq2XwS58sER5GcLLLQv26pPL8hgu6CdI0sHxgcgQBgpjoLwUMHmv13CnN/AAcZY4RRDK2X5zd6wFSEhuuCoUqh/m7K219IxI6fVkLaApC851D/ua/pj9OMjiOaJ+D13cPrgfqxva4eK8FQZvqQm7aUK/TT216QNYmn7hkzNjMTkjv/L5yYQ5Ym92AHy44wHzLPrtDB5mk94vHOjG+KlAt9ofuLncMSAXWe+fOqxuIQ6VfbL2LUv0EGYhHe4XOVj98q5utsVQxQbXAZzNQaq9FWXs2A4xcVP7aWj0Fooi17LN/W0ng4Iijir5hyLr7k7M1G6resgiY7p83biNow== pdonnell@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQCpgLVt7Zphmf6nqD+9BC0N91jdmaY6ZpvrYWlzbhxXITQ6juIEtegnlZ+8ohdeEfHFBiyu1xOSQSM1X1ugvpPQE94UqQg7JVb0tuoUCNJHYnIuv8T4YgyP0/vEV48k2G0BBWlP+IJ5f1iDvZuLHWLR5JSgGqh5ELaaHr7Cbyb6JlImldb7SDIoPHMCZ+n6dLO8l5Zy7qWelw6aeA770xmPucBr6t/BVns2g77XgJz6zFd99yEaDy27i8jfzL7WqR9i93yVE9EsL7tuOjg6G9KUzxBF0bg9iVsJXypvcBuG0JwQ+SDKqYw/cN4iLy5qBa6Siqrfpc/LKU21VNmJ0e5eXNgKKbMBpBb8GVmkoLaSRcXw7LclWopjNDA+meUhDAwU7b97N7cqN8BYhBKQyiShXFHMKLuNo2fSuTRt/icfb9VisKNGBdvwgV0oOlrqbXMzZKmj9amb9b+3JDBTGB7ioCsj5x/Evf9srXRrQ9kikIr1aBbVXIWir/lasNWVydcmUYsNi8L7W0vXArLwp+GgOXJFeZUiD3z9LG4CouRxMSwoj3lgWeL2ZR8feH4D/SP/+QmBdU6+aG97au9qreeXi5X8QTR3LND0DwwEKczX+3s+Fa950Oieom/gXC57PVKVh90KVoAGw3HzJTlZrauOtLkYbTUAX6woOMhAtJ5oWuDDT4UrHQj37jRrrl2dETLwKwUJN2+2L07vnnOZ4ABcGWLsU6d3bXeUBlw3pdsYqPebDKUEpjL2kUOw+zmfjE 2021-11-11T20:38:58.483 INFO:teuthology.task.ansible.out:fdVx9BfrvExvDX+3MkxccCJ8FY7od1s5fGEFHsXnIiMnLO/HZb/JN+Aoe+/ExadIcYwJnWcCJyly2/532kGFUlzOliCwPxgJpjWCKQAv6LjWmxoPXiA4JI0bp6y7lMEjmV2Bp3mwphlsW/r8Ie5hg1UPh/ttHRp/VZgcV8e6nNmP/qc6k9b2DWTNFqtWqm1hXLOwq08HzUEeH9mTqvenyAbv2RFMOS18Y2K+4BKUzHwZujrz9qEh/kqtpFScEIXkOJ+HZz3mI6XfjWFIPS487yRGfUvW3+nSWhvHc8vJY21bFRKdjD84qCVlZ2BfOCGUZSKftVItEiqXMTRs3RZd5epfCVZbJuA0hG16jUseeHqqZqDtlbLkzkxtS6v3iGRnPh/Ke/94FHHCzvYY8cmMYAVaM6p/G3edWftbaEtN70Fo7VKxPrpPzuSRZ7Vxmnb+Zad95DxxAC/vtMy/vkBVIP6dyn4HGxqa1oECFMe4oXdhnxZQoPzvWxb8upcXeAeOMMbcIlMq5a+dCBPGFNKhNhrb+9uAEDHwjE7rZlMJVSvt9v1S4hll6jiYIap+7n4reSKrS45aO9U0xiKXJJDohh pdonnell@icewind\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQC8MdUJzIvTCn0fM+hv1HAO/Lwjy/C/V7stQRWGIxyrRHbniyX8iuaU3j7YNFglSQPJq+YRM9UnGauM9/0J9tHKLsMp+zRQYrtxU/ThiTUN8TRJY11mGUol4yLsqkTtm6oq5sl6o+LfuA4hzaYc3XLrFHRJuCUgbtPG71Q0C/y3U+itrDK4wXUxBa4YzZzcIMg+c88VXwoouPbiL0gW5PCXzMaNtKlOqBUKlE935MLn70wIEfN/EwAAbss0ga1moqeqmi9Nlvm20DNIVFmkAGtYoq6CHXtL/uehzai5Sueop8Vmd8+1U8gDTo6JpiagdD8ZNf0zAbRx587KAT5ZMnahJpO276j7RNa85ZFkNkEnBLqBHRkCkTCYq1GehYyRaR69uuKyczFiIoTfAQgPh7VlnpT9525b/4nFsBDOkWxxWAIJR61YiKemPV/n/e9SwhvVsMDb2B/RBriwR7Mi2SZ/lBHGB+z0OA+EE2rGZy4LFVOfUczqIdpltlNotXan4M4F361i+bX19KE+f2RLySn3+GiFv0CdLwslS/y4xysz+FWxw6B0Y/EkpiDU3CdWM61+THGGm/DVhfL/Pvca7k8PonHFYw/x5w1nBcADpkM9JF+m5eOnp+weTEzl58sS+3GnoAuL8YFJwMW1RmE50xM0o5E3SDc8XftUvaSZz8mhII3uwDDukbELfmkLbnIeipQKyjO+GEx0JBp4Itz82qGuao0Wi32U9wHzhSWnX4GQ0pKZnq9frSlyCNwHu/iMiKF8sA2bX3dvplFJ9nKWPefcU8bdl/upr5hYrd2GttjUSkWZa+UuF264yx3ztcN7jotRhEmmdEcHVzKQ26/NyRI4/9DQpDKY7zw4O2wFkXWg5L9kodXwWzgy4cyOr4dMD9LBVLOpHc0CAji5/+lfuYQbu1mwVdfOzCXDqByjt7zCbeN7KuUh403sJ8PTQVp642xA9i4AkPv5g+fqi5C0++1DbZP0lmLfKtTswHhoN+kAqZu2N4Mu6mNVNKyZgW5h1pZzNSGWYcZttyYmMCHhPketReL0LPJ+VUsqQCVohuta3aaSe44InA28GZsXzMj5B0el/84bnE7cN4b+rfGD25NT73JEt3RgPa9W9jqT1hwd/nf53BDKpp7DlHjR77WSWT4h4F7bhbM1RsGSqAfZjCUJT6Rq+IuM2XcrLDY8zIeArkqaZLXMzfGj6KJ7cQes05P5LYfY4osXiq68Gs/7gzF31PQRRGOYAoqMN7752sn0ZMcpnUj3mJutQcncRUN4+naTMjU36WWD7it0YuQBnb2IJ4iKjYLInw8pr2zKgWtgr1TSCM4xKp/v1/ZMmbBmS+d 2021-11-11T20:38:58.485 INFO:teuthology.task.ansible.out:TzNC7Mpkt8vdCNjzZ5KZ5 pdonnell@aglarond", "key_options": null, "keyfile": "/home/pdonnell/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pdonnell", "validate_certs": true} 2021-11-11T20:38:58.685 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jlayton', 'ovpn': 'jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jlayton", "ovpn": "jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDKzhyWb3PmU88m8BbgpEuBzYM6+x28z6aZwHSesiyfXMfF4Y7IZjtKaTxH0ph/O9GQCPhRCVhRd8TlgKWJGhts1E+VkQfHfT18CgmrPWzj3gifc2FJhL97vB1Pd3jksGnOPAikgYPHP46SVMTtyvXCkOUuo7E29WrkdpjawcqM1rfpw63OnzHf7Ar7Sqh/othB7ajyVQ92t+XWSEqj7K+NRfTaq5L5RBWbwarM885o8xdI5mimOZ0QjzKOhFDORrI+HEVlkutyB8E20nZgo7TUy+kYeLVhEqb3A6Pi2QbA9GFLT69kA/H9nebhj7Cuim9yrx4OKeZuIncG0UxSQF3QmWflAZHBs8L/jniTz4DJ5id5jR29tvGK2LeddCEMB1L2cHwWHlG0aX5jaNEJe+nG6H5Tcg3+M1RPM8twBkRQmS3UzAyX/MTH1kktyFUHn8CdBXe0WTuoFSvXLB6BCHWzDYZOt0tIV/xMtBwhy+XGYA4oZSsDU5m7VlSVLBGL8DbRX69Z8AGey2y3FoFu/roY6NmDhTxUedJ2ZfoKSwdZD6jjatdO1621nonBnJxrDgaXCWzT7HtnWcc3GwxZoBrqkvgpsx88f4lNi22OpYVPV6A4hECe04u6OJnHSWVk3ft3p5PaLNF8aEXbiIgzZKAwX3LxpvEDFH/t1sdihhpmjw== jlayton@synchrony.poochiereds.net", "key_options": null, "keyfile": "/home/jlayton/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jlayton", "validate_certs": true} 2021-11-11T20:38:58.717 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jlayton', 'ovpn': 'jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jlayton", "ovpn": "jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDKzhyWb3PmU88m8BbgpEuBzYM6+x28z6aZwHSesiyfXMfF4Y7IZjtKaTxH0ph/O9GQCPhRCVhRd8TlgKWJGhts1E+VkQfHfT18CgmrPWzj3gifc2FJhL97vB1Pd3jksGnOPAikgYPHP46SVMTtyvXCkOUuo7E29WrkdpjawcqM1rfpw63OnzHf7Ar7Sqh/othB7ajyVQ92t+XWSEqj7K+NRfTaq5L5RBWbwarM885o8xdI5mimOZ0QjzKOhFDORrI+HEVlkutyB8E20nZgo7TUy+kYeLVhEqb3A6Pi2QbA9GFLT69kA/H9nebhj7Cuim9yrx4OKeZuIncG0UxSQF3QmWflAZHBs8L/jniTz4DJ5id5jR29tvGK2LeddCEMB1L2cHwWHlG0aX5jaNEJe+nG6H5Tcg3+M1RPM8twBkRQmS3UzAyX/MTH1kktyFUHn8CdBXe0WTuoFSvXLB6BCHWzDYZOt0tIV/xMtBwhy+XGYA4oZSsDU5m7VlSVLBGL8DbRX69Z8AGey2y3FoFu/roY6NmDhTxUedJ2ZfoKSwdZD6jjatdO1621nonBnJxrDgaXCWzT7HtnWcc3GwxZoBrqkvgpsx88f4lNi22OpYVPV6A4hECe04u6OJnHSWVk3ft3p5PaLNF8aEXbiIgzZKAwX3LxpvEDFH/t1sdihhpmjw== jlayton@synchrony.poochiereds.net", "key_options": null, "keyfile": "/home/jlayton/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jlayton", "validate_certs": true} 2021-11-11T20:38:59.091 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rzarzynski', 'ovpn': 'rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rzarzynski", "ovpn": "rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCijPApnekNl/dsY+fleKYstEzNkKflBEcCoKtjItUCDJCVLVFOsDBrV94EGvDWpr13aj9sSwpUzRbeu/vOW8ZT853HH6Dqg2+eoYwG6vDRprZb93nw6l4vtU59/M0yKkyi9CPp2+YUo31cCFzYWpq8bv2j1xBruDSUCUdn9nJrHiLFqA4QEwSpw+uVCjSKzQL+LjlKmE8Plfd+8mNCQeY/OFsjxAztwN+4M50SZCnOVCIwEXoYRNd82JOXMOaqZJazSWTO8Tml6+Nu0hyEcvHkHxWRDSM+6LmXNnjlVUPyDN2h1/VAYATpOZ2VhmRarDoftCp9Wzi7eQdL/CGiMyuD rzarzyns@redhat.com", "key_options": null, "keyfile": "/home/rzarzynski/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rzarzynski", "validate_certs": true} 2021-11-11T20:38:59.102 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rzarzynski', 'ovpn': 'rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rzarzynski", "ovpn": "rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCijPApnekNl/dsY+fleKYstEzNkKflBEcCoKtjItUCDJCVLVFOsDBrV94EGvDWpr13aj9sSwpUzRbeu/vOW8ZT853HH6Dqg2+eoYwG6vDRprZb93nw6l4vtU59/M0yKkyi9CPp2+YUo31cCFzYWpq8bv2j1xBruDSUCUdn9nJrHiLFqA4QEwSpw+uVCjSKzQL+LjlKmE8Plfd+8mNCQeY/OFsjxAztwN+4M50SZCnOVCIwEXoYRNd82JOXMOaqZJazSWTO8Tml6+Nu0hyEcvHkHxWRDSM+6LmXNnjlVUPyDN2h1/VAYATpOZ2VhmRarDoftCp9Wzi7eQdL/CGiMyuD rzarzyns@redhat.com", "key_options": null, "keyfile": "/home/rzarzynski/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rzarzynski", "validate_certs": true} 2021-11-11T20:38:59.320 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rdias', 'ovpn': 'rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rdias", "ovpn": "rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDdGthG4qx90VQ8mJ7TI/IFFzptkSqBHYZsFIVu06IQgHbnFBvjao6heGt4UlkWJkegPCDWpKVAe7wFuXDpPlj/VssRHB5LBnYHKGfkNAzMNiMjpEZsTdy+9w39CCujipz0AlOSfrUdANBZ11AnG9VgQSPEukX5487juob55lFM6J3x+v4opBF3rSlux3+thbr8V0w4oCHMD9/aHvVjFoBnxo4oirR4jc/jgC2fQCMiAJhFKJGhjkKFiNHxcB3npBuRnpy+59BcScOAzSXdJ7DGnwxQLjymnhqsAcWinjjJhOjaq6NaoeKv5I7zQLaVGaQdIyuuAK1Dnb7trpfQU7mR rdias@rdias-suse-laptop", "key_options": null, "keyfile": "/home/rdias/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rdias", "validate_certs": true} 2021-11-11T20:38:59.331 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rdias', 'ovpn': 'rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rdias", "ovpn": "rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDdGthG4qx90VQ8mJ7TI/IFFzptkSqBHYZsFIVu06IQgHbnFBvjao6heGt4UlkWJkegPCDWpKVAe7wFuXDpPlj/VssRHB5LBnYHKGfkNAzMNiMjpEZsTdy+9w39CCujipz0AlOSfrUdANBZ11AnG9VgQSPEukX5487juob55lFM6J3x+v4opBF3rSlux3+thbr8V0w4oCHMD9/aHvVjFoBnxo4oirR4jc/jgC2fQCMiAJhFKJGhjkKFiNHxcB3npBuRnpy+59BcScOAzSXdJ7DGnwxQLjymnhqsAcWinjjJhOjaq6NaoeKv5I7zQLaVGaQdIyuuAK1Dnb7trpfQU7mR rdias@rdias-suse-laptop", "key_options": null, "keyfile": "/home/rdias/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rdias", "validate_certs": true} 2021-11-11T20:38:59.549 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vshankar', 'ovpn': 'vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vshankar", "ovpn": "vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDEEjb7yrcY6I6XbMU0hCps1NqBuYmA6HDmMcS/l3i5wJRBitIXYVlv6pefUG9gdBYR5NneccLfVFURu+DC63MJa2ELJvVXsQ6YUOX39bJUT9KlZs8mH+nlTD7tlPmP17Ae3k8sT0H42Q+/5+b5YvdAU+03aPhzntmy38SyvLKrvowRk63wHgcE9p23LY9tY3+HSJu86cb/IdePk1GLjnrMeT9aM+5j2eETrmBjMArP+/2UIsT5eb8MBkQzJ299SCC2gfV/qIVkorFl9xdejctAwNwdT0SPTMxCHxaiK3wZ0e/5HHTbSgR5YlxOlKYq93XBVYIaFKPCePk1At1a3Pjh cephalopod@octopus\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcrTZoKO5e5xs2PB2UuAruvGlJESTHNAtqipbC8yJtRg8WGVDLt//rGu36QNSbpbJYMZgqgKsXBYolJB1ckqLYLv5j8RAoVwbW4QqKaV95y7x0bbXs/YAPnRBhEO1nfpT8GHQusUZvTbz38FDaQwstsYoncmsyZwBrqus9ycFavpY4wfpKoWtzeiK5Wjq7ervThOs98ZJKXJXOOxeI7mDakE3Jv7q2tBNu/rU1kzxIl65kWURzgxox6EmIZXQK3FzKt0jjdKtApVYSlx4ODLPI4iyxOv+MbRe3Nnj6FDhfz8/FeP/U5wWxdYx2WslzZCOI1BJ0VDLu3xEpLlu4vPzp vshankar@h3ckers-pride", "key_options": null, "keyfile": "/home/vshankar/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vshankar", "validate_certs": true} 2021-11-11T20:38:59.570 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vshankar', 'ovpn': 'vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vshankar", "ovpn": "vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDEEjb7yrcY6I6XbMU0hCps1NqBuYmA6HDmMcS/l3i5wJRBitIXYVlv6pefUG9gdBYR5NneccLfVFURu+DC63MJa2ELJvVXsQ6YUOX39bJUT9KlZs8mH+nlTD7tlPmP17Ae3k8sT0H42Q+/5+b5YvdAU+03aPhzntmy38SyvLKrvowRk63wHgcE9p23LY9tY3+HSJu86cb/IdePk1GLjnrMeT9aM+5j2eETrmBjMArP+/2UIsT5eb8MBkQzJ299SCC2gfV/qIVkorFl9xdejctAwNwdT0SPTMxCHxaiK3wZ0e/5HHTbSgR5YlxOlKYq93XBVYIaFKPCePk1At1a3Pjh cephalopod@octopus\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcrTZoKO5e5xs2PB2UuAruvGlJESTHNAtqipbC8yJtRg8WGVDLt//rGu36QNSbpbJYMZgqgKsXBYolJB1ckqLYLv5j8RAoVwbW4QqKaV95y7x0bbXs/YAPnRBhEO1nfpT8GHQusUZvTbz38FDaQwstsYoncmsyZwBrqus9ycFavpY4wfpKoWtzeiK5Wjq7ervThOs98ZJKXJXOOxeI7mDakE3Jv7q2tBNu/rU1kzxIl65kWURzgxox6EmIZXQK3FzKt0jjdKtApVYSlx4ODLPI4iyxOv+MbRe3Nnj6FDhfz8/FeP/U5wWxdYx2WslzZCOI1BJ0VDLu3xEpLlu4vPzp vshankar@h3ckers-pride", "key_options": null, "keyfile": "/home/vshankar/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vshankar", "validate_certs": true} 2021-11-11T20:38:59.788 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'akupczyk', 'ovpn': 'adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "akupczyk", "ovpn": "adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+CtnvyVRSkIvD15O/q7/vQmD4oE8q7Mo6oT3kqTXaEaQ7iDayC2biUC0PC4OztPlNJxPUcKK5DfWw4F79OKeheqUbZRhsCC3Ge2JGoXHmsf0kBYdioMGiAxm+f8M/v1KESSSxvEnU7o+oi6VtFchh4Dl/WZ54rqZc/oQrlGNTHulFAuyoIAlKkCe3N0GdHN54PB+26QbLXvOuqpWmjVFbcjgRC62mkZl3LuCBhviXS4Rl5/ZE0QIHubCwV5XVrsqvscLS64bi06W8RN2vL56Vn3N9TGV4d2dlbE1PaBBESoTQ7Q0iUN32OAZLo7nF/1fiPY3Xc8r35juya5k+KR9z adam@TP50", "key_options": null, "keyfile": "/home/akupczyk/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "akupczyk", "validate_certs": true} 2021-11-11T20:38:59.799 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'akupczyk', 'ovpn': 'adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "akupczyk", "ovpn": "adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+CtnvyVRSkIvD15O/q7/vQmD4oE8q7Mo6oT3kqTXaEaQ7iDayC2biUC0PC4OztPlNJxPUcKK5DfWw4F79OKeheqUbZRhsCC3Ge2JGoXHmsf0kBYdioMGiAxm+f8M/v1KESSSxvEnU7o+oi6VtFchh4Dl/WZ54rqZc/oQrlGNTHulFAuyoIAlKkCe3N0GdHN54PB+26QbLXvOuqpWmjVFbcjgRC62mkZl3LuCBhviXS4Rl5/ZE0QIHubCwV5XVrsqvscLS64bi06W8RN2vL56Vn3N9TGV4d2dlbE1PaBBESoTQ7Q0iUN32OAZLo7nF/1fiPY3Xc8r35juya5k+KR9z adam@TP50", "key_options": null, "keyfile": "/home/akupczyk/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "akupczyk", "validate_certs": true} 2021-11-11T20:39:00.027 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'nojha', 'ovpn': 'nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "nojha", "ovpn": "nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDeUJcLeyfqIk3ErPoFqh9kOnvUPsk/2hfaYNC2wZLMOSTIs3zXM8xjki8eN6vle0dmmeeAdwF+e2hJ3tCFKsEUQRvMc7ugIBA7L8/DboShYu7ItyQ3QRvy9pi6jnA/tlHA1LHalvyhMw0yZ4kvPbrfNMhZzOKzvL6jX166rqLA5AuiIfdYMBpE5rYq11GVjm0uQKw4UJpBMooNJeovEfQlNBTSV+IJSTiGBpBD7SgsuW3USMeEF3fEgQOcnvYdv5Fbp/crloyLWYc0HTidkLxpjP7vqY7E4BuBJug0zIB7voR3MYq6rAeuWjByPvq6coDlUjnw/QlrvarCbjjSaHdaRXLj/Dj+QqJFviESBr64ZynhHL21Bl6rexW7AwYyPa73Erbus3dN0aVQQ/ApcDTGHT7KRlDQ2YYg05YM4swpVVnrIdV58FzLWkll+ypCK/+kdtk5eRSe/KzZA3s7z4d4WLgolFWs8Fky8Rx1xDGewvMJgFUWVUbcGC13RocbNSxXG80cbIx8L/sclX4lDfGouu2sUXhP19vq1rJ7gZa6XTPohXjPbFS+En2cdlGdKXhppdvjT4QWKNzgQoF7qgWHMqJnMFJ6beT/2zP34KoM3t0ulee04z88VSn8Th8aqvLlGbdrt5LrE90escSaIAR1vNKZ1hCFwGieSKoRtYJTCw== nojha@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6NWn9EzQJoeXkUbaNclYcR9Gga7HFhhXMGxdo/9ypYwlEzQTA6FnEreG3s1EmlQ4hofuc0kMhrjh0YeJZMrETWMoA0SmoX+6iR26UWVtOTk7i1v2TqkTY7llmQneHqQgmZEBXK0jF1HVP5kHY5dhagtcpfsGY5dc0eGUjfqAyh/xrCxciOE/JQ03tDcKIc5eOQIqv46JqnUS90Zr/Cov2qGKKHTa/VO1a2viOmDC/uHi5xXvII1Kh9rHp1oZ2Wm3fiGgT8xZaTJFFhWpdq3AdKfkEaV93yQcCdaUsGBHwDHcPOhxpBZpqZvcCR3I/tigDav4QViB1PRN7JnTIWCIznfrQK3r2Frl6BVFDvydwgRVS43ctEKPOJj8ELMUWX+fpjsKrA9Kn7nUU35ti7qIWKg1Cnz3lzVA29TsEg10E+nT4SVKZtRSWw/2AX0bsfxfSY30yjw5PP+bjnVh/bOOkqR2UkPBj4eeOKMD8nV1an79xP0eH+t36INaJhq6qwxE= nojha@nojha-mac", "key_options": null, "keyfile": "/home 2021-11-11T20:39:00.029 INFO:teuthology.task.ansible.out:/nojha/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "nojha", "validate_certs": true} ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'nojha', 'ovpn': 'nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "nojha", "ovpn": "nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDeUJcLeyfqIk3ErPoFqh9kOnvUPsk/2hfaYNC2wZLMOSTIs3zXM8xjki8eN6vle0dmmeeAdwF+e2hJ3tCFKsEUQRvMc7ugIBA7L8/DboShYu7ItyQ3QRvy9pi6jnA/tlHA1LHalvyhMw0yZ4kvPbrfNMhZzOKzvL6jX166rqLA5AuiIfdYMBpE5rYq11GVjm0uQKw4UJpBMooNJeovEfQlNBTSV+IJSTiGBpBD7SgsuW3USMeEF3fEgQOcnvYdv5Fbp/crloyLWYc0HTidkLxpjP7vqY7E4BuBJug0zIB7voR3MYq6rAeuWjByPvq6coDlUjnw/QlrvarCbjjSaHdaRXLj/Dj+QqJFviESBr64ZynhHL21Bl6rexW7AwYyPa73Erbus3dN0aVQQ/ApcDTGHT7KRlDQ2YYg05YM4swpVVnrIdV58FzLWkll+ypCK/+kdtk5eRSe/KzZA3s7z4d4WLgolFWs8Fky8Rx1xDGewvMJgFUWVUbcGC13RocbNSxXG80cbIx8L/sclX4lDfGouu2sUXhP19vq1rJ7gZa6XTPohXjPbFS+En2cdlGdKXhppdvjT4QWKNzgQoF7qgWHMqJnMFJ6beT/2zP34KoM3t0ulee04z88VSn8Th8aqvLlGbdrt5LrE90escSaIAR1vNKZ1hCFwGieSKoRtYJTCw== nojha@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6NWn9EzQJoeXkUbaNclYcR9Gga7HFhhXMGxdo/9ypYwlEzQTA6FnEreG3s1EmlQ4hofuc0kMhrjh0YeJZMrETWMoA0SmoX+6iR26UWVtOTk7i1v2TqkTY7llmQneHqQgmZEBXK0jF1HVP5kHY5dhagtcpfsGY5dc0eGUjfqAyh/xrCxciOE/JQ03tDcKIc5eOQIqv46JqnUS90Zr/Cov2qGKKHTa/VO1a2viOmDC/uHi5xXvII1Kh9rHp1oZ2Wm3fiGgT8xZaTJFFhWpdq3AdKfkEaV93yQcCdaUsGBHwDHcPOhxpBZpqZvcCR3I/tigDav4QViB1PRN7JnTIWCIznfrQK3r2Frl6BVFDvydwgRVS43ctEKPOJj8ELMUWX+fpjsKrA9Kn7nUU35ti7qIWKg1Cnz3lzVA29TsEg10E+nT4SVKZtRSWw/2AX0b 2021-11-11T20:39:00.030 INFO:teuthology.task.ansible.out:sfxfSY30yjw5PP+bjnVh/bOOkqR2UkPBj4eeOKMD8nV1an79xP0eH+t36INaJhq6qwxE= nojha@nojha-mac", "key_options": null, "keyfile": "/home/nojha/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "nojha", "validate_certs": true} 2021-11-11T20:39:00.258 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'ifed01', 'ovpn': 'ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ifed01", "ovpn": "ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+c7zV5QmZb0WsgA2UZogoDNaoHcngRVLA8+BYk/U10ZlvCckJSfIXWMlBnqguZgdm/jBwsw5mGVBgHxUSqFNybwIlU00kyPZU6VbyVP+ZNVUf0XIE1NlemldgPBxxcOzSHkHhWrUqA3IKiCllcQVDQzutfUnq+jDT6NkVn4/cyNzNlargt7FR2L8gvskh62jLTBbhEuHMBp3jLCvaHJOk708gXm9qZ5iLOGV0pu7xdg58KE0p8ILClQBTbvoqPgZE9LA3qaOk1NR1n22kuW8JRHaKfD2Hn0Yu/MfKct783Bxb3mxWBKXxmM4otdmsR/R14NH/hIPce9wSkOOnQAC9", "key_options": null, "keyfile": "/home/ifed01/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ifed01", "validate_certs": true} 2021-11-11T20:39:00.259 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'ifed01', 'ovpn': 'ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ifed01", "ovpn": "ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+c7zV5QmZb0WsgA2UZogoDNaoHcngRVLA8+BYk/U10ZlvCckJSfIXWMlBnqguZgdm/jBwsw5mGVBgHxUSqFNybwIlU00kyPZU6VbyVP+ZNVUf0XIE1NlemldgPBxxcOzSHkHhWrUqA3IKiCllcQVDQzutfUnq+jDT6NkVn4/cyNzNlargt7FR2L8gvskh62jLTBbhEuHMBp3jLCvaHJOk708gXm9qZ5iLOGV0pu7xdg58KE0p8ILClQBTbvoqPgZE9LA3qaOk1NR1n22kuW8JRHaKfD2Hn0Yu/MfKct783Bxb3mxWBKXxmM4otdmsR/R14NH/hIPce9wSkOOnQAC9", "key_options": null, "keyfile": "/home/ifed01/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ifed01", "validate_certs": true} 2021-11-11T20:39:00.487 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'myoungwon', 'ovpn': 'myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "myoungwon", "ovpn": "myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCnOq6OHTRJuv7n0GZygBue/AE2q47pFM3KyrvmZxPRNmwtXn2r3ngHQQyc6DLZdFIv9Yn0VWu1KB/ryCdN3Xy7HAxBfHdKjtQ4vvfsWUbQF2JjhEjrsl+usgpqK1Tz6yuL4uAtFgLTN9o6gcH2bsQ15w+rEHCiYVzaGp7Cj+y+/VPBRIEHETSUZxwqiDH57uFTHxJrBIn6yhCorbn5QOjfmHUGmkxl3Geph++LbuQeha5F/DHaylMulsYZZBDEhIG8G2coDmVz8Cp1gw0kBWv63JHKnXaJboGXty2Zj36O/wdZKTE+7cV6JhxfJczhZBTb2cImIhspGwdHfnFgBoXz myoungwon@fg2server", "key_options": null, "keyfile": "/home/myoungwon/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "myoungwon", "validate_certs": true} 2021-11-11T20:39:00.489 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'myoungwon', 'ovpn': 'myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "myoungwon", "ovpn": "myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCnOq6OHTRJuv7n0GZygBue/AE2q47pFM3KyrvmZxPRNmwtXn2r3ngHQQyc6DLZdFIv9Yn0VWu1KB/ryCdN3Xy7HAxBfHdKjtQ4vvfsWUbQF2JjhEjrsl+usgpqK1Tz6yuL4uAtFgLTN9o6gcH2bsQ15w+rEHCiYVzaGp7Cj+y+/VPBRIEHETSUZxwqiDH57uFTHxJrBIn6yhCorbn5QOjfmHUGmkxl3Geph++LbuQeha5F/DHaylMulsYZZBDEhIG8G2coDmVz8Cp1gw0kBWv63JHKnXaJboGXty2Zj36O/wdZKTE+7cV6JhxfJczhZBTb2cImIhspGwdHfnFgBoXz myoungwon@fg2server", "key_options": null, "keyfile": "/home/myoungwon/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "myoungwon", "validate_certs": true} 2021-11-11T20:39:00.720 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'gabrioux', 'ovpn': 'gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "gabrioux", "ovpn": "gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDZPRW12u7LvaiynB87ad5h/JFtYIVjkKIOym9Y5CAbvatHhZ9fB1WAnYIn2HqiSSPUXT8sFy8w0TmM+aH0/ZGcfXoe0CdTWLtUbXKNlPI5h7TO9rHXESP2iRBpnKs3vK8WnMY0/D5BeESrPfwVScucmzaDrjUqjdZKR/aEeqVvDpQsdaCaoqNg23t4nhQ54BEvK8O+14nQ26Rtmm40QHO1ze+0UK+7nUMMFwxvd1EKjVp21EgYAFEbyQ5Kbbl+fmS/11AGy/yTai0c4qd7ohBzZvx8+sY+c1novG9Q7LhtyxuuYwzYIrGBCkKH5+iC7nt2rGBP+mnTfIgdbJUiXX3cdrMESwOSZy6x6zgbXXME+yfM0myhomKmUkpeZAQInX2KYrnEuPD9kteyK9tZcG6gJSaiUdvlab49Alx5lM2hc62M4s4WKaIgroCoApd8FtNAlTQsdYuycntgNrbPy0LXQ5hw8K8hkvc6RbxISdj/RXWguQSVbJWdS9Pf5NwD3Tq7dsOghJWhx4vBkhW3IECVrPQWVyAmgQ9oj1PoL7/2LZ6h9P8F0KC1qmhm+pWdEPLugOlU6lzQX7RJcukYlsINThM1RGwbPE4h9XcMyHAnODiDF63TDfQ01oKbVXj9XpOGY5eP0EQxKJYjCfWW0Ot6nK2ZCt8OYvoRkWMGnLv11Nub3NNOgh5TB/4nDvEU1cIb/Ikb4BRzVIEbast/pqUJmXe+GQ+ho7mp8y8a+lIxu54BR+VN6isjwODCVt1DYohgixEoul7p5pXvTM+JUY2GD/ddTGwPvVAN66sWnT9bjZZunQwTHqasFCIHNyPhnEnYIIqCo9O1I/D8zRQG3SYNxT6ED9GPARrE/H0TvdOmKaT3e0F2GRJRSToNxol9dpJovUziojOhXyBUjlTqnMY2RVilX90MnzgqR3zGO0aSruTGyNT2Faymqxn6T/50mSdJ8EyWH23BaaXUUW/z3Cjov+yCQjLY48SOWwlxKXw3hGz8xTAxauBYI01ojsVLCpCUGFyKlWS5iFwiLFtr1KPYqeCoS6CoWjiOiFgGe+w3lPPfp2h4rc6auIx2DZ1ByYhWLeHnpkV7sG+q5ckBIRAYmiKyXSGdV4zbXITMWDm+3zu4oj5VPRCfjMlout8OZ4TQm5PUdKZZhSE1jULG9tPDFL7nWwM3z0HET+d9kQYcEO6QB+ma9gZHp1qZJbXvCIC5fQ9y9El++A6N3dBr+ED10Ye88PLoWPeumyAmc24BDdI0skGk89Q0IzoLqagK6HGhPAso8si9wr84CEi4ncpizVLDU2cFKCpef4Sa03HoMbwfTgERtyptKFRbMT+SpuZsnrhF635h9uqFnszowUXV guits@Elisheba", "key_options": null, "keyfile": "/home/gabrioux/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": " 2021-11-11T20:39:00.722 INFO:teuthology.task.ansible.out:gabrioux", "validate_certs": true} 2021-11-11T20:39:00.731 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'gabrioux', 'ovpn': 'gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "gabrioux", "ovpn": "gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDZPRW12u7LvaiynB87ad5h/JFtYIVjkKIOym9Y5CAbvatHhZ9fB1WAnYIn2HqiSSPUXT8sFy8w0TmM+aH0/ZGcfXoe0CdTWLtUbXKNlPI5h7TO9rHXESP2iRBpnKs3vK8WnMY0/D5BeESrPfwVScucmzaDrjUqjdZKR/aEeqVvDpQsdaCaoqNg23t4nhQ54BEvK8O+14nQ26Rtmm40QHO1ze+0UK+7nUMMFwxvd1EKjVp21EgYAFEbyQ5Kbbl+fmS/11AGy/yTai0c4qd7ohBzZvx8+sY+c1novG9Q7LhtyxuuYwzYIrGBCkKH5+iC7nt2rGBP+mnTfIgdbJUiXX3cdrMESwOSZy6x6zgbXXME+yfM0myhomKmUkpeZAQInX2KYrnEuPD9kteyK9tZcG6gJSaiUdvlab49Alx5lM2hc62M4s4WKaIgroCoApd8FtNAlTQsdYuycntgNrbPy0LXQ5hw8K8hkvc6RbxISdj/RXWguQSVbJWdS9Pf5NwD3Tq7dsOghJWhx4vBkhW3IECVrPQWVyAmgQ9oj1PoL7/2LZ6h9P8F0KC1qmhm+pWdEPLugOlU6lzQX7RJcukYlsINThM1RGwbPE4h9XcMyHAnODiDF63TDfQ01oKbVXj9XpOGY5eP0EQxKJYjCfWW0Ot6nK2ZCt8OYvoRkWMGnLv11Nub3NNOgh5TB/4nDvEU1cIb/Ikb4BRzVIEbast/pqUJmXe+GQ+ho7mp8y8a+lIxu54BR+VN6isjwODCVt1DYohgixEoul7p5pXvTM+JUY2GD/ddTGwPvVAN66sWnT9bjZZunQwTHqasFCIHNyPhnEnYIIqCo9O1I/D8zRQG3SYNxT6ED9GPARrE/H0TvdOmKaT3e0F2GRJRSToNxol9dpJovUziojOhXyBUjlTqnMY2RVilX90MnzgqR3zGO0aSruTGyNT2Faymqxn6T/50mSdJ8EyWH23BaaXUUW/z3Cjov+yCQjLY48SOWwlxKXw3hGz8xTAxauBYI01ojsVLCpCUGFyKlWS5iFwiLFtr1KPYqeCoS6CoWjiOiFgGe+w3lPPfp2h4rc6auIx2DZ1ByYhWLeHnpkV7sG+q5ckBIRAYmiKyXSGdV4zbXITMWDm+3zu4oj5VPRCfjMlout8OZ4TQm5PUdKZZhSE1jULG9tPDFL7nWwM3z0HET+d9kQYcEO6QB+ma9gZHp1qZJbXvCIC5fQ9y9El++A6N3dBr+ED10Ye88PLoWPeumyAmc24BDdI0skGk89Q0IzoLqagK6HGhPAso8si9wr84CEi4ncpizVLDU2cFKCpef4Sa03HoMbwfTgERtyptKFRbMT+SpuZsnrhF635h9uqFnszowUXV guits@Elisheba", "key_options": null, "keyfile": "/home/gabrioux/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": " 2021-11-11T20:39:00.732 INFO:teuthology.task.ansible.out:gabrioux", "validate_certs": true} 2021-11-11T20:39:00.949 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'leseb', 'ovpn': 'leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "leseb", "ovpn": "leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCpDFqu0Zg94mCjrA0FTB0vMiSBml1pqYCDKHhKBrfaza8u3bhtsmls9i+6U/vIwCEVk4GGPNhWduTJK0h9R5MS2eCf3GJcE0XHqRp/8msUCDPOVSZwC7hoYHZKunUIUaIUajOnR+L2slA/a9JaqataFH7zdrKbPdSKBzf5CXYrc0fK+Fz5OOsaI9KITbM0nESVpNCDzRqnA/aarF9MlN79vpTtvwQ+i0z2Km3cxqwV70GPMuFVo4CGf8pCGpbMrIg9OtoSC1svQ2vZzLkH2WrbDqTfYlOEYJJXaiBKcGhGnLZXfLIGSgvHjpv6xHo63yaMihocyg2MwTS8TZ/VtAaH seb@redhat.com", "key_options": null, "keyfile": "/home/leseb/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "leseb", "validate_certs": true} 2021-11-11T20:39:00.960 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'leseb', 'ovpn': 'leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "leseb", "ovpn": "leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCpDFqu0Zg94mCjrA0FTB0vMiSBml1pqYCDKHhKBrfaza8u3bhtsmls9i+6U/vIwCEVk4GGPNhWduTJK0h9R5MS2eCf3GJcE0XHqRp/8msUCDPOVSZwC7hoYHZKunUIUaIUajOnR+L2slA/a9JaqataFH7zdrKbPdSKBzf5CXYrc0fK+Fz5OOsaI9KITbM0nESVpNCDzRqnA/aarF9MlN79vpTtvwQ+i0z2Km3cxqwV70GPMuFVo4CGf8pCGpbMrIg9OtoSC1svQ2vZzLkH2WrbDqTfYlOEYJJXaiBKcGhGnLZXfLIGSgvHjpv6xHo63yaMihocyg2MwTS8TZ/VtAaH seb@redhat.com", "key_options": null, "keyfile": "/home/leseb/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "leseb", "validate_certs": true} 2021-11-11T20:39:01.188 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'hchen', 'ovpn': 'hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "hchen", "ovpn": "hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAzjJ6nlPn/OXP3knQ6orVId2QLswql9gRetKYlicKDBrdFg0MES03EIuAonu0aUR4NEReR7P1iB8eNnezrHyAu5mxIJlZp0rz7ljcBcTLL+11IJU9I9OjBFwzXGqJ1KK4B4Qfw35oc2O6F5V0JP2nHhHgyny37udJLImhxHy3RNmDwc86JhEBCVR3ouYiZuxgxxbA9BXc6+Lkdu9hEMMCDlaEXwNmMyASVU9Wf3dKnMTSCdar2PANquWbpNQZ5cgoTm39xgust5CEyYJ53aOdrti0DYK7qyzxUXre6ckZGrH4oGv2XrDaxpACIiHXt+slvow51gsZnBRHeayMAxQzGQ== hchen@hchen.csb", "key_options": null, "keyfile": "/home/hchen/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "hchen", "validate_certs": true} 2021-11-11T20:39:01.189 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'hchen', 'ovpn': 'hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "hchen", "ovpn": "hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAzjJ6nlPn/OXP3knQ6orVId2QLswql9gRetKYlicKDBrdFg0MES03EIuAonu0aUR4NEReR7P1iB8eNnezrHyAu5mxIJlZp0rz7ljcBcTLL+11IJU9I9OjBFwzXGqJ1KK4B4Qfw35oc2O6F5V0JP2nHhHgyny37udJLImhxHy3RNmDwc86JhEBCVR3ouYiZuxgxxbA9BXc6+Lkdu9hEMMCDlaEXwNmMyASVU9Wf3dKnMTSCdar2PANquWbpNQZ5cgoTm39xgust5CEyYJ53aOdrti0DYK7qyzxUXre6ckZGrH4oGv2XrDaxpACIiHXt+slvow51gsZnBRHeayMAxQzGQ== hchen@hchen.csb", "key_options": null, "keyfile": "/home/hchen/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "hchen", "validate_certs": true} 2021-11-11T20:39:01.418 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jcollin', 'ovpn': 'jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jcollin", "ovpn": "jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCqZKc+/7iPLhapA/7SYmLxwyKEM95D6eFfsDBNKbEUey/B1OlqYSo+yKB/LdFDyUcd7shMDnhFFg47lNMANrsZdtE4iMPVAa1YhcrTgNbV3HSp9cthKixvU/gkt9Fnb6PEckRzr19wEwxtoNg3Fh/R6YCiSjPwqHmArtC1G2hzk560EJpozx6xylMR5IafatNDoZVw2YQ35fZvHbhHD6meQELducHJ1bKEEHxVIRSTcmXDpBz/vAvBoQOEPuH6PmC/oSwL9HVVIsazf65F4QfK98d+8NCK9TemgXyGDk3Sjz6xdm76yBhFzDQEAzO12U1YtS7Q7zGVsjmSDav2dd7 jcollin@earth\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC89dHSsgjgtIjJ3e/pcMthliQlX2vtdXL7pk62nD64230Fm1PfOVCnzn4LqDhBbGgJT+BrlwOPfzStpLgnUT3l67S834SJzhvSTTpQp2pznnQg1rZOiXjvNhbQwVYV4rphXWqay8weWvFIb5QfXE66mYA1etZtcOb1SMY2ofAjUiA8bAQS5aHpzcMVs4N1A9x5GJx/7/Bj1h8zPpSXikDYasJc3++UMGc45C/AAf725YwijK7xMnZNXv8uTBoGzO/Ko24R3VdvtfMVn4Hi4/FwuRd5bRYcJXR/nUdtA3IinqvlnTpFV2LbfgVTHiUMGhvewRTB+8S8ij4nnAlko3H3bhqZZTk/y7d4K/IerY0mozVIc2x8rMLfqhF9OKv1QFB9fskRZcF/ENHITnmLqCJPg9rgmyv1gCQ5Tbds1OI+IYQIowdiwtaViiTSIy7/CedbHqMz6aDnPsJFMs24lcUXe2d8uZ26RqMinMFGXg0zF7tifuVmxj7zG8DT7MFORDm5f/Hhkng05R5XZesStTBxpEU0MGuzWhENt3RuJdstqmY+wkDVW3XwjUvKG/gVYuJYfnBAsRZIZezw2wfvodhc8EmeHELvY1YtB3DGTBXKE1lsKGN7yV5+nl4UtR2IGzBn9GovWVi5ooB8UlSAeUmZix/eAA5teL8ldFrDht7+AQ== jcollin@teuthology\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDeApVHu5er+R9h52RLSVYVhXFQSn/nMN38uWoDYPw2bqYBLNEAAMcPl+rnqHubELpXTGXjJYL4n2mjic+2rOLZpDgPLmOOm4PwgYstJjE1KLegKqVzXIa9HfASKMXaHSgwtC6P5vDQkm2MNl8vXuTAITF7VYdDS33Kk 2021-11-11T20:39:01.420 INFO:teuthology.task.ansible.out:ke6ViJRZI7szYjbP7z9A1fy26THrvnuBT1Q18yy9vgk/sKo8vjLQfHSLvO90CJzouNEKqmkVpmJEqCPbE+s6Qr4+d2KY7L6Q6GAXxrqXK1jSDrrPsJguE6zZ1oWy/7XFMOr9ssduFWocnjQjPtQj3M+V+7FYAsf5uqQnzeMQTjO6AXYxTI7 jcollin@stratocaster\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDk/DYRp5E8Dla/4XHCF504X20W0u7CijZSCFWT1/5082NgnYeR90Prth3p+4xsw3YC9H5pEAS3XwcbdUb/NtWvZVIAIcMd7NOOEClSIj2Agk7NJxZ/4SfQRLrK3H2Qli/MVnrTxnEa9oeS4zqzv9dKRwFJDUFCoH2CpoumFVAp02uyzk7D/+XqFrXzMJUiSHXfl3YM2rt/vNmCtHtVp04UcG0Kyl2TXY37frRlSOkMMu0zbIC551gOyae6NGYhkYHa7u2SnngXcKM78b8MDvMv3QtuUN5P4WpH5qFRdlKNKsQGrzcQpBDY8AKQ+rX0K1ENhBwer49bihYNMKvjYmvTdLTNWgPCu7dt8JSgJQ8JMMhve0pjvFdpcKDLp0fvXDvJvZdLJM1ID/66kywJspHjShuqQNXRMfd1R9C4F20c+R9LY6eTGwrpLG1DNEez1VBHmOu5hQl03T2PfaThfLfMHRTw8JpGXGLMQarCCTm1PC6Z2Iw7KzorxP1eofGA5q8XoCbwGD01uQRZrwxPq0SHzTAkW2C2M8oEz+k4Mrx7+vGYzSS14hqxrzk91wuYhZCAKuGpqwm1ouJ4wxvWFewjbxLgucMfGSp09zJ2XSYRmtm7SAUifFob6Z9czHGTQerHCOTPuYqzlap2gCCVYtEvQyNB+4IqK/9FtEXTr28x0Q== jcollin@stratocaster", "key_options": null, "keyfile": "/home/jcollin/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jcollin", "validate_certs": true} 2021-11-11T20:39:01.429 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jcollin', 'ovpn': 'jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jcollin", "ovpn": "jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCqZKc+/7iPLhapA/7SYmLxwyKEM95D6eFfsDBNKbEUey/B1OlqYSo+yKB/LdFDyUcd7shMDnhFFg47lNMANrsZdtE4iMPVAa1YhcrTgNbV3HSp9cthKixvU/gkt9Fnb6PEckRzr19wEwxtoNg3Fh/R6YCiSjPwqHmArtC1G2hzk560EJpozx6xylMR5IafatNDoZVw2YQ35fZvHbhHD6meQELducHJ1bKEEHxVIRSTcmXDpBz/vAvBoQOEPuH6PmC/oSwL9HVVIsazf65F4QfK98d+8NCK9TemgXyGDk3Sjz6xdm76yBhFzDQEAzO12U1YtS7Q7zGVsjmSDav2dd7 jcollin@earth\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC89dHSsgjgtIjJ3e/pcMthliQlX2vtdXL7pk62nD64230Fm1PfOVCnzn4LqDhBbGgJT+BrlwOPfzStpLgnUT3l67S834SJzhvSTTpQp2pznnQg1rZOiXjvNhbQwVYV4rphXWqay8weWvFIb5QfXE66mYA1etZtcOb1SMY2ofAjUiA8bAQS5aHpzcMVs4N1A9x5GJx/7/Bj1h8zPpSXikDYasJc3++UMGc45C/AAf725YwijK7xMnZNXv8uTBoGzO/Ko24R3VdvtfMVn4Hi4/FwuRd5bRYcJXR/nUdtA3IinqvlnTpFV2LbfgVTHiUMGhvewRTB+8S8ij4nnAlko3H3bhqZZTk/y7d4K/IerY0mozVIc2x8rMLfqhF9OKv1QFB9fskRZcF/ENHITnmLqCJPg9rgmyv1gCQ5Tbds1OI+IYQIowdiwtaViiTSIy7/CedbHqMz6aDnPsJFMs24lcUXe2d8uZ26RqMinMFGXg0zF7tifuVmxj7zG8DT7MFORDm5f/Hhkng05R5XZesStTBxpEU0MGuzWhENt3RuJdstqmY+wkDVW3XwjUvKG/gVYuJYfnBAsRZIZezw2wfvodhc8EmeHELvY1YtB3DGTBXKE1lsKGN7yV5+nl4UtR2IGzBn9GovWVi5ooB8UlSAeUmZix/eAA5teL8ldFrDht7+AQ== jcollin@teuthology\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDeApVHu5er+R9h52RLSVYVhXFQSn/nMN38uWoDYPw2bqYBLNEAAMcPl+rnqHubELpXTGXjJYL4n2mjic+2rOLZpDgPLmOOm4PwgYstJjE1KLegKqVzXIa9HfASKMXaHSgwtC6P5vDQkm2MNl8vXuTAITF7VYdDS33Kk 2021-11-11T20:39:01.431 INFO:teuthology.task.ansible.out:ke6ViJRZI7szYjbP7z9A1fy26THrvnuBT1Q18yy9vgk/sKo8vjLQfHSLvO90CJzouNEKqmkVpmJEqCPbE+s6Qr4+d2KY7L6Q6GAXxrqXK1jSDrrPsJguE6zZ1oWy/7XFMOr9ssduFWocnjQjPtQj3M+V+7FYAsf5uqQnzeMQTjO6AXYxTI7 jcollin@stratocaster\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDk/DYRp5E8Dla/4XHCF504X20W0u7CijZSCFWT1/5082NgnYeR90Prth3p+4xsw3YC9H5pEAS3XwcbdUb/NtWvZVIAIcMd7NOOEClSIj2Agk7NJxZ/4SfQRLrK3H2Qli/MVnrTxnEa9oeS4zqzv9dKRwFJDUFCoH2CpoumFVAp02uyzk7D/+XqFrXzMJUiSHXfl3YM2rt/vNmCtHtVp04UcG0Kyl2TXY37frRlSOkMMu0zbIC551gOyae6NGYhkYHa7u2SnngXcKM78b8MDvMv3QtuUN5P4WpH5qFRdlKNKsQGrzcQpBDY8AKQ+rX0K1ENhBwer49bihYNMKvjYmvTdLTNWgPCu7dt8JSgJQ8JMMhve0pjvFdpcKDLp0fvXDvJvZdLJM1ID/66kywJspHjShuqQNXRMfd1R9C4F20c+R9LY6eTGwrpLG1DNEez1VBHmOu5hQl03T2PfaThfLfMHRTw8JpGXGLMQarCCTm1PC6Z2Iw7KzorxP1eofGA5q8XoCbwGD01uQRZrwxPq0SHzTAkW2C2M8oEz+k4Mrx7+vGYzSS14hqxrzk91wuYhZCAKuGpqwm1ouJ4wxvWFewjbxLgucMfGSp09zJ2XSYRmtm7SAUifFob6Z9czHGTQerHCOTPuYqzlap2gCCVYtEvQyNB+4IqK/9FtEXTr28x0Q== jcollin@stratocaster", "key_options": null, "keyfile": "/home/jcollin/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jcollin", "validate_certs": true} 2021-11-11T20:39:01.647 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'xxg', 'ovpn': 'xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "xxg", "ovpn": "xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDdxi9xdXNBYctMHH1cuuyNcPC4I4xVAeD3HZsKH+QwIILiKOTHu/6HZg7Yo1WsuwYSA0MDc+5LCSGu7RsmOEVpcSQ4gtNoFVwsThDyI6yhsm9WQqqXwSPs7ZE+urqd/BDfQ/7oMqJm5bViQMP0vmBExHuIFagj+8f6nspkE2Z7lBbaw8bEbAJZFx74oReOAELsL+Pbtf9Iaqrap4Z3WysmhfP6Rm/q7mncBBmo/fl/bejhThr8fghTPJb81M+XLhMk/rAdVrGdKV9iSt6lLiWdvf0lpTZO4LdIfJSBXr1Xv+lHRtaGXID2ahH7zl/W6dM4baz+hU7ZrbP+vdKg/SVH xxg@xxg", "key_options": null, "keyfile": "/home/xxg/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "xxg", "validate_certs": true} 2021-11-11T20:39:01.658 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'xxg', 'ovpn': 'xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "xxg", "ovpn": "xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDdxi9xdXNBYctMHH1cuuyNcPC4I4xVAeD3HZsKH+QwIILiKOTHu/6HZg7Yo1WsuwYSA0MDc+5LCSGu7RsmOEVpcSQ4gtNoFVwsThDyI6yhsm9WQqqXwSPs7ZE+urqd/BDfQ/7oMqJm5bViQMP0vmBExHuIFagj+8f6nspkE2Z7lBbaw8bEbAJZFx74oReOAELsL+Pbtf9Iaqrap4Z3WysmhfP6Rm/q7mncBBmo/fl/bejhThr8fghTPJb81M+XLhMk/rAdVrGdKV9iSt6lLiWdvf0lpTZO4LdIfJSBXr1Xv+lHRtaGXID2ahH7zl/W6dM4baz+hU7ZrbP+vdKg/SVH xxg@xxg", "key_options": null, "keyfile": "/home/xxg/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "xxg", "validate_certs": true} 2021-11-11T20:39:01.887 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'pcuzner', 'ovpn': 'pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pcuzner", "ovpn": "pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsA63hT3ZAwQ9hsvfY+o2Gy15M1GRnRw1ZBSXEiCs9OJku7T9+LCvvH85Y/7V+mOGcsCZKqVTd5Vy3sHAvAF3M7CYJ+5IIDJKCYslC0wkKcX4jj26aMToAZPms8uvs527BNgKOcvwgIfPjs4WJG06fey5AhsuuL30etbAvbBudxRAKdZSTOsMveneKtXQu3Dx7y0rjfI/RBAhCAvisnMqYFUdajoJIl5gVXl6+8mauUOP5O0mvyvJH2jQveqZXTdsIPyp5mq1nA5psqvivWGz2bEJ+owI0GjfuYbyAi6i+BL4Bsd3GeoAs1G3Dicfr228VIUsT+R3xLBQUj3xIsVAZ pcuzner@redhat.com", "key_options": null, "keyfile": "/home/pcuzner/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pcuzner", "validate_certs": true} 2021-11-11T20:39:01.889 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'pcuzner', 'ovpn': 'pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pcuzner", "ovpn": "pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsA63hT3ZAwQ9hsvfY+o2Gy15M1GRnRw1ZBSXEiCs9OJku7T9+LCvvH85Y/7V+mOGcsCZKqVTd5Vy3sHAvAF3M7CYJ+5IIDJKCYslC0wkKcX4jj26aMToAZPms8uvs527BNgKOcvwgIfPjs4WJG06fey5AhsuuL30etbAvbBudxRAKdZSTOsMveneKtXQu3Dx7y0rjfI/RBAhCAvisnMqYFUdajoJIl5gVXl6+8mauUOP5O0mvyvJH2jQveqZXTdsIPyp5mq1nA5psqvivWGz2bEJ+owI0GjfuYbyAi6i+BL4Bsd3GeoAs1G3Dicfr228VIUsT+R3xLBQUj3xIsVAZ pcuzner@redhat.com", "key_options": null, "keyfile": "/home/pcuzner/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pcuzner", "validate_certs": true} 2021-11-11T20:39:02.116 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mkogan', 'ovpn': 'mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mkogan", "ovpn": "mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDaRX+aBLjb/h12ctmvoMvZro6NrrYDuxROF6QMVpEVucA7jHhwCC/j9dJAq0hOVBO8R3S0bvKQ6bmnCb/4y2J6tq18xKUFCrChNii5OQ+AgM5v1353oZ9SrHN8p9ajvS8w313o2OpXhpaPv3khVDjKZsDmeH8YL/Nx3WQKqIJC1mI4jrI0sZGGJLifEy3pPkO0W1KmguN9AwAWK6+kckAjMa3ElkfK/xuAqBYgNKQUxP9oMgTQRcgpWF8Nzb0npN3v+HOUIRrth1INSchkDKtWob2T+0Zqxwm+Vpi0iFWwqjcN/OVdaDGf34//A3M/tsiQ1g9SqhdhZJCfu5XLmM63 mkogan@mkP50.redhat.com", "key_options": null, "keyfile": "/home/mkogan/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mkogan", "validate_certs": true} 2021-11-11T20:39:02.118 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mkogan', 'ovpn': 'mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mkogan", "ovpn": "mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDaRX+aBLjb/h12ctmvoMvZro6NrrYDuxROF6QMVpEVucA7jHhwCC/j9dJAq0hOVBO8R3S0bvKQ6bmnCb/4y2J6tq18xKUFCrChNii5OQ+AgM5v1353oZ9SrHN8p9ajvS8w313o2OpXhpaPv3khVDjKZsDmeH8YL/Nx3WQKqIJC1mI4jrI0sZGGJLifEy3pPkO0W1KmguN9AwAWK6+kckAjMa3ElkfK/xuAqBYgNKQUxP9oMgTQRcgpWF8Nzb0npN3v+HOUIRrth1INSchkDKtWob2T+0Zqxwm+Vpi0iFWwqjcN/OVdaDGf34//A3M/tsiQ1g9SqhdhZJCfu5XLmM63 mkogan@mkP50.redhat.com", "key_options": null, "keyfile": "/home/mkogan/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mkogan", "validate_certs": true} 2021-11-11T20:39:02.345 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vumrao", "ovpn": "vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC56FiAgzz9aF+qAMkw5aGoCyoSA8Lf2LPaz3QxTYEzmQt0ByXMmVw0cgUI4gDt7hMBX6LoxPALZ7C1aH8aPTpb6+hxHGfir2tqQmU49JOQiUvBP/Xtt1ilxMou9Wu/ZadEhIZxaP/TBncXNls9cbFs9bV7Q8J3CLCTGRa8kilEpZ7209c/xQI4yQ6q+Q3H4FkO0YHP+Tkn9mg1df+BKkXfSBycjAyOXPvljoetjw0FHEbJ7lTCcSG+hCb2LEACo2OK9eWC3GYwDPKZF1Ht9HfDvg4bHfY3OVOixYiZSim7dbj3+VHWWsiH64usGH1o6fn4vV16UhVJiiicPMWvllzH vikhyat.umrao@gmail.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC9G/I5iaw5d4ssyJ4iTUxSvIBAX+P03VVHsNMHAB70rVGGwBnBJFqJNbukzxfnsqwzN6vSeuMy7I1zc2VVi4GtBQPK0iCbVQN+Njq0VCIxb4hLmjD3Vn8NgRp80955VJyYGT9+/JbZY1UCdPBuld6hsnTipkdo5U3huAHD4i1FpYKol+4DCZHAO5wNilCbests0+ZPtHYe53vnNDz685cx/f7RTtEarvU2fECxkurlBBNvuIl6SN8bo+NaJ01Li0xAD+2fpGz6Iu0WUEdWTkt78ubvA4If2k8XAKxkOKEvfyX+AtTzdHxWSQu7ixkbuksuK6A0bnankn+DdiP6Y293 vumrao@redhat", "key_options": null, "keyfile": "/home/vumrao/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vumrao", "validate_certs": true} 2021-11-11T20:39:02.346 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vumrao", "ovpn": "vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC56FiAgzz9aF+qAMkw5aGoCyoSA8Lf2LPaz3QxTYEzmQt0ByXMmVw0cgUI4gDt7hMBX6LoxPALZ7C1aH8aPTpb6+hxHGfir2tqQmU49JOQiUvBP/Xtt1ilxMou9Wu/ZadEhIZxaP/TBncXNls9cbFs9bV7Q8J3CLCTGRa8kilEpZ7209c/xQI4yQ6q+Q3H4FkO0YHP+Tkn9mg1df+BKkXfSBycjAyOXPvljoetjw0FHEbJ7lTCcSG+hCb2LEACo2OK9eWC3GYwDPKZF1Ht9HfDvg4bHfY3OVOixYiZSim7dbj3+VHWWsiH64usGH1o6fn4vV16UhVJiiicPMWvllzH vikhyat.umrao@gmail.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC9G/I5iaw5d4ssyJ4iTUxSvIBAX+P03VVHsNMHAB70rVGGwBnBJFqJNbukzxfnsqwzN6vSeuMy7I1zc2VVi4GtBQPK0iCbVQN+Njq0VCIxb4hLmjD3Vn8NgRp80955VJyYGT9+/JbZY1UCdPBuld6hsnTipkdo5U3huAHD4i1FpYKol+4DCZHAO5wNilCbests0+ZPtHYe53vnNDz685cx/f7RTtEarvU2fECxkurlBBNvuIl6SN8bo+NaJ01Li0xAD+2fpGz6Iu0WUEdWTkt78ubvA4If2k8XAKxkOKEvfyX+AtTzdHxWSQu7ixkbuksuK6A0bnankn+DdiP6Y293 vumrao@redhat", "key_options": null, "keyfile": "/home/vumrao/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vumrao", "validate_certs": true} 2021-11-11T20:39:02.566 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'henrix', 'ovpn': 'henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "henrix", "ovpn": "henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDZDKRAePFqjugc2ljw4QmM6r7B2dPI39OBB+2HKONtDZI4q+uwgZ1yqnhrbJaU9ux0W5wNzJyp2VfOPhw/9G9jo+efJPg5IySnTonhi1zeCXNpBMY70dJZVCyGEDdWKehvFM9xFBOxBC/h7gOyKqEc28BKoy4cHfULzm4er8PU30MyFBngu8NaoKqyQ3GYAfWOAXF5lUdalbJwl3a+HBb8PjUhbGuznAZF1Ji4JgnNuEZdlNVOKwhEkAFZ1JwXvgLfrEJVQs5b2V9Sp2Vm5ZngsLg7b3RzV5I4B3Lj7QTTraV1zLJGiS5iRdjY7D4W5Uxi2k1X7vaOjySJWWadHyeicNNBhLSfHl6JJ3aMuA8oAAmtgo/8IkJ96GFtgM9j31Wg8wkeTyQrwSeqCuv1uDFr8rp+OvMClhHnDgUklJLWjjYPCDVYR5KV+iOiHL58WJg/b4h6NsoP91wO8P8EcSaiMdq7P77w5nJxDI0KKb//dC8aHtWnRRRlrIS+PNPqKJSNTaSwOTieXvzplhzKFzu2dr79qs+/67iaNE6HBvIX5jVHWCFV3wllyjsC8WYPh4kEVtg2SY00SDOOrMrqANV8DDtHFwwxQTEjz1z5ESuwy9UcUSQfP1/zsAp2OQzPutKs6UipHyBK3pLm6MjaeeaEWu8fmetCqALxYV3h17O+Yw== miguel@hermes.olymp", "key_options": null, "keyfile": "/home/henrix/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "henrix", "validate_certs": true} 2021-11-11T20:39:02.577 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'henrix', 'ovpn': 'henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "henrix", "ovpn": "henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDZDKRAePFqjugc2ljw4QmM6r7B2dPI39OBB+2HKONtDZI4q+uwgZ1yqnhrbJaU9ux0W5wNzJyp2VfOPhw/9G9jo+efJPg5IySnTonhi1zeCXNpBMY70dJZVCyGEDdWKehvFM9xFBOxBC/h7gOyKqEc28BKoy4cHfULzm4er8PU30MyFBngu8NaoKqyQ3GYAfWOAXF5lUdalbJwl3a+HBb8PjUhbGuznAZF1Ji4JgnNuEZdlNVOKwhEkAFZ1JwXvgLfrEJVQs5b2V9Sp2Vm5ZngsLg7b3RzV5I4B3Lj7QTTraV1zLJGiS5iRdjY7D4W5Uxi2k1X7vaOjySJWWadHyeicNNBhLSfHl6JJ3aMuA8oAAmtgo/8IkJ96GFtgM9j31Wg8wkeTyQrwSeqCuv1uDFr8rp+OvMClhHnDgUklJLWjjYPCDVYR5KV+iOiHL58WJg/b4h6NsoP91wO8P8EcSaiMdq7P77w5nJxDI0KKb//dC8aHtWnRRRlrIS+PNPqKJSNTaSwOTieXvzplhzKFzu2dr79qs+/67iaNE6HBvIX5jVHWCFV3wllyjsC8WYPh4kEVtg2SY00SDOOrMrqANV8DDtHFwwxQTEjz1z5ESuwy9UcUSQfP1/zsAp2OQzPutKs6UipHyBK3pLm6MjaeeaEWu8fmetCqALxYV3h17O+Yw== miguel@hermes.olymp", "key_options": null, "keyfile": "/home/henrix/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "henrix", "validate_certs": true} 2021-11-11T20:39:02.805 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'pbs1108', 'ovpn': 'pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pbs1108", "ovpn": "pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNwmgnIsDDJmsmYzkTAEo3iLj7Jd5Ni2R5vD1Z0NRZ39zJMtbicZ0ec3nAMzFyHrmPnKNFltS1mhehbLbtKNKq1fSo2SMHs11z+N8Nl94F4AY8IhXZ8U9GAorEJFVIxHUNEDqwl55Ov6MfP60UI7rNgy2rUt8aOBmAJVhLEFmJJawJgyPhTrUZAT20OoILphrt81o5bTNfmx++Zs5Dc9VtgIh53hexwKgTvG6fxfKdra9m+J2GXpniOsZjPJq/A0je+t7H3UlxCSu0dRy2Un6l5ZAcoeg74SP50dFgGaxkTQqI6wle6NjtPIdssgrdYTgTcOg9M4jWqAr/5drTUSGh root@centos7-bspark", "key_options": null, "keyfile": "/home/pbs1108/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pbs1108", "validate_certs": true} 2021-11-11T20:39:02.807 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'pbs1108', 'ovpn': 'pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pbs1108", "ovpn": "pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNwmgnIsDDJmsmYzkTAEo3iLj7Jd5Ni2R5vD1Z0NRZ39zJMtbicZ0ec3nAMzFyHrmPnKNFltS1mhehbLbtKNKq1fSo2SMHs11z+N8Nl94F4AY8IhXZ8U9GAorEJFVIxHUNEDqwl55Ov6MfP60UI7rNgy2rUt8aOBmAJVhLEFmJJawJgyPhTrUZAT20OoILphrt81o5bTNfmx++Zs5Dc9VtgIh53hexwKgTvG6fxfKdra9m+J2GXpniOsZjPJq/A0je+t7H3UlxCSu0dRy2Un6l5ZAcoeg74SP50dFgGaxkTQqI6wle6NjtPIdssgrdYTgTcOg9M4jWqAr/5drTUSGh root@centos7-bspark", "key_options": null, "keyfile": "/home/pbs1108/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pbs1108", "validate_certs": true} 2021-11-11T20:39:03.034 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'clacroix', 'ovpn': 'clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "clacroix", "ovpn": "clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDnvVnbK2SNfL8hTmrA09L41RudKW/F7Ty0vvlHjrFjDkdWXVgrafFWGKQZ8dSR6FJ9JLU0X59xiEo4z31ini/Dt2XozFMIr7RdeB3xpiifuUUG5Lgsasd2ezniUoOEukL1a9KkCIxNgmKJuvtlHCt+ZTtyL513yjRICByZbnmBSSpUiett50X10vtvaNBu4LikUJKLc8gP0PHLbaGsU3uOUd655lkFWjWhgExlU8bHWiAZq18Ss63jDlSKFVSk903GKSNcHXMHNPYaXlrYw8NFzejoy/XjxEfmF/Lv3MeAUhHQWx25sh5aDv48mzo8FaIJsL1MrD0QxF1gllobWAfp clacroix@redhat.com", "key_options": null, "keyfile": "/home/clacroix/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "clacroix", "validate_certs": true} 2021-11-11T20:39:03.035 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'clacroix', 'ovpn': 'clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "clacroix", "ovpn": "clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDnvVnbK2SNfL8hTmrA09L41RudKW/F7Ty0vvlHjrFjDkdWXVgrafFWGKQZ8dSR6FJ9JLU0X59xiEo4z31ini/Dt2XozFMIr7RdeB3xpiifuUUG5Lgsasd2ezniUoOEukL1a9KkCIxNgmKJuvtlHCt+ZTtyL513yjRICByZbnmBSSpUiett50X10vtvaNBu4LikUJKLc8gP0PHLbaGsU3uOUd655lkFWjWhgExlU8bHWiAZq18Ss63jDlSKFVSk903GKSNcHXMHNPYaXlrYw8NFzejoy/XjxEfmF/Lv3MeAUhHQWx25sh5aDv48mzo8FaIJsL1MrD0QxF1gllobWAfp clacroix@redhat.com", "key_options": null, "keyfile": "/home/clacroix/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "clacroix", "validate_certs": true} 2021-11-11T20:39:03.263 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'epuertat', 'ovpn': 'epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "epuertat", "ovpn": "epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAyiGmj+nOAD+il+FvXvZSpUOJQp6hKh6UPaAIQvQZkdeNRWN7wcDXMUlLj+TBFWa1OO+9xqFlnbgVMDR4qy5orUBYlMapUb1rutjbk64XeY58b6uguHwSoMd81QpCCIN4KOk3cldV8UiyX382baoZ9Je2hsOnnBGMsRtHuVHZ90Bj1rIITkwajq//KRs7kjb/qhjk0xkipDIcaSwKjzPlmmnYn69noODwDexpvLYlCF+LowSZEdrOUvVIigSFG7cLTDaU2tLhxSgbvWu8f0pwvjsyW7gOgM+VeRZ6uHC79F1qvgdgQSxflddIgrQpH0rzf6RNxQjR3SBkZ4V1BESa0w== epuertat@epuertat", "key_options": null, "keyfile": "/home/epuertat/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "epuertat", "validate_certs": true} 2021-11-11T20:39:03.275 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'epuertat', 'ovpn': 'epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "epuertat", "ovpn": "epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAyiGmj+nOAD+il+FvXvZSpUOJQp6hKh6UPaAIQvQZkdeNRWN7wcDXMUlLj+TBFWa1OO+9xqFlnbgVMDR4qy5orUBYlMapUb1rutjbk64XeY58b6uguHwSoMd81QpCCIN4KOk3cldV8UiyX382baoZ9Je2hsOnnBGMsRtHuVHZ90Bj1rIITkwajq//KRs7kjb/qhjk0xkipDIcaSwKjzPlmmnYn69noODwDexpvLYlCF+LowSZEdrOUvVIigSFG7cLTDaU2tLhxSgbvWu8f0pwvjsyW7gOgM+VeRZ6uHC79F1qvgdgQSxflddIgrQpH0rzf6RNxQjR3SBkZ4V1BESa0w== epuertat@epuertat", "key_options": null, "keyfile": "/home/epuertat/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "epuertat", "validate_certs": true} 2021-11-11T20:39:03.492 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'tdehler', 'ovpn': 'tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "tdehler", "ovpn": "tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDRZyWiDO5JqWHJdUUBr8YReiKXxnL87juFAfkcmw2E9KmANVH2sn3K+OgUCQfBs1Wdg6AbcL2GyMm9alodUruzOm127/bH9OcGmu+8SNsMAYrAF9EzsKJa8Ba/I05rl5a87p1jbYG9Rpo7u3OvLx0LjgqydRYcmjwo/niifYNDTaN0K3v8s1mN4TorUZW/i3XBMBIBjlIiNGs/I7IZreR+1N4V8Nwrp2EWxDv7AwG3nvokL2RTHcsV0KgmAnciv8nXXjIf+uqOPuGA+F2qW/wz92bG8R0O4CjH+QZt4k6UoUIXFRW+LdOVAqmUrb4W0OJ8vsVeukW46SvPnYUTkZnF sonea@think", "key_options": null, "keyfile": "/home/tdehler/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "tdehler", "validate_certs": true} 2021-11-11T20:39:03.504 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'tdehler', 'ovpn': 'tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "tdehler", "ovpn": "tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDRZyWiDO5JqWHJdUUBr8YReiKXxnL87juFAfkcmw2E9KmANVH2sn3K+OgUCQfBs1Wdg6AbcL2GyMm9alodUruzOm127/bH9OcGmu+8SNsMAYrAF9EzsKJa8Ba/I05rl5a87p1jbYG9Rpo7u3OvLx0LjgqydRYcmjwo/niifYNDTaN0K3v8s1mN4TorUZW/i3XBMBIBjlIiNGs/I7IZreR+1N4V8Nwrp2EWxDv7AwG3nvokL2RTHcsV0KgmAnciv8nXXjIf+uqOPuGA+F2qW/wz92bG8R0O4CjH+QZt4k6UoUIXFRW+LdOVAqmUrb4W0OJ8vsVeukW46SvPnYUTkZnF sonea@think", "key_options": null, "keyfile": "/home/tdehler/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "tdehler", "validate_certs": true} 2021-11-11T20:39:03.721 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'laura', 'ovpn': 'laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "laura", "ovpn": "laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDEBD7CxJJfbLW4jtuLnTwkxVbhCfh/JL3Qj8oeLEkny1tE7QCeXn1bP5bScWSgLH34Wh4TAuH+8/g/koFwcCy1YuoGBdBV4KV7O9MTnFngdqNY6LKzCipm27tnWZEa10hrF8dRFTmhracd6kmmVGHgvGQPVAVONmix+2IZ83wdj8XJSQHeE3PHFLO39tU+ez/T76QaiBLDVo4ctHVt+N9bBbHV6Cr8fyiUUU6YEKMlcedvxGpnMK8oeOSNuDeh+DldcM9KYzNGEOM34jGRIvJp57dxQDQCtPPW5hvBXg7yMyrv6TcJzDd2LFD6AC/Qw9yrEQPXXyBCDkAlCqDR0+rH", "key_options": null, "keyfile": "/home/laura/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "laura", "validate_certs": true} 2021-11-11T20:39:03.733 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'laura', 'ovpn': 'laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "laura", "ovpn": "laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDEBD7CxJJfbLW4jtuLnTwkxVbhCfh/JL3Qj8oeLEkny1tE7QCeXn1bP5bScWSgLH34Wh4TAuH+8/g/koFwcCy1YuoGBdBV4KV7O9MTnFngdqNY6LKzCipm27tnWZEa10hrF8dRFTmhracd6kmmVGHgvGQPVAVONmix+2IZ83wdj8XJSQHeE3PHFLO39tU+ez/T76QaiBLDVo4ctHVt+N9bBbHV6Cr8fyiUUU6YEKMlcedvxGpnMK8oeOSNuDeh+DldcM9KYzNGEOM34jGRIvJp57dxQDQCtPPW5hvBXg7yMyrv6TcJzDd2LFD6AC/Qw9yrEQPXXyBCDkAlCqDR0+rH", "key_options": null, "keyfile": "/home/laura/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "laura", "validate_certs": true} 2021-11-11T20:39:03.961 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'yaarit', 'ovpn': 'yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "yaarit", "ovpn": "yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2k8xsrsLxEqJx3rV6ZZwLH0OjYCDB2uIYzcAyrKEOKlVWO1ICKxkw+gZ9tSiPFNLfq13io9BIrAKJwlrdLCJLmlgCQwpwUQqzmRlRZEm+WmD0dl8LPeQSidyJkPkJgYBEshFPjDHIj10Ei1pJS5svI2l7G1wMWxlt2kHL8Og/4zdFTRctxwqZcvzLmRQcs8lSSpqTkhfeKfmrpR+je8XFbCJJ9m/nJbgyCiZKaGN8nf36pCQDEva4rvaSrK0a6YxL0RWN8W3lqtBPKIehxffr3lYDEkD9EUW5vf6AUKC0ioh4jkPo9WCjaoFsOauMOMVQPHehEAMufEO4Kym6nfrf yaarit@localhost.localdomain", "key_options": null, "keyfile": "/home/yaarit/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "yaarit", "validate_certs": true} 2021-11-11T20:39:03.963 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'yaarit', 'ovpn': 'yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "yaarit", "ovpn": "yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2k8xsrsLxEqJx3rV6ZZwLH0OjYCDB2uIYzcAyrKEOKlVWO1ICKxkw+gZ9tSiPFNLfq13io9BIrAKJwlrdLCJLmlgCQwpwUQqzmRlRZEm+WmD0dl8LPeQSidyJkPkJgYBEshFPjDHIj10Ei1pJS5svI2l7G1wMWxlt2kHL8Og/4zdFTRctxwqZcvzLmRQcs8lSSpqTkhfeKfmrpR+je8XFbCJJ9m/nJbgyCiZKaGN8nf36pCQDEva4rvaSrK0a6YxL0RWN8W3lqtBPKIehxffr3lYDEkD9EUW5vf6AUKC0ioh4jkPo9WCjaoFsOauMOMVQPHehEAMufEO4Kym6nfrf yaarit@localhost.localdomain", "key_options": null, "keyfile": "/home/yaarit/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "yaarit", "validate_certs": true} 2021-11-11T20:39:04.189 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rpavani1998', 'ovpn': 'admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rpavani1998", "ovpn": "admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDUznqo3VDmcKRL64P7sbjuxXCW/0ljFTc5zZoZ9UvtZvU2jrRQ2X9GyxcrcQnAWlF5YZHqeTDU5o8ixn5fq1MSmaPRBNDKKVQOfR/vFQV73LiKx4BFsC1/EdaPie/XqltaLDTtfqhxADJlz1H3a3ZlkeVHMROTqo/TidL1YUYlRH4qHxWWBuFuVS+EYfTegSZdjwp+DuJA0iN6X99+DfH2DhiHENjEmlbKN13+H+EPzm0K3ya2w8D6ANodB3AxMn2tGs8mLtG/P4kCjoDnoUVFzKTyUpTtgOjis47uSY4dL8mTbIK9kZAiFxnT5sMClu6KRStDjRIrxbTBQJ8wvlat admin1@rajula", "key_options": null, "keyfile": "/home/rpavani1998/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rpavani1998", "validate_certs": true} 2021-11-11T20:39:04.191 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rpavani1998', 'ovpn': 'admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rpavani1998", "ovpn": "admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDUznqo3VDmcKRL64P7sbjuxXCW/0ljFTc5zZoZ9UvtZvU2jrRQ2X9GyxcrcQnAWlF5YZHqeTDU5o8ixn5fq1MSmaPRBNDKKVQOfR/vFQV73LiKx4BFsC1/EdaPie/XqltaLDTtfqhxADJlz1H3a3ZlkeVHMROTqo/TidL1YUYlRH4qHxWWBuFuVS+EYfTegSZdjwp+DuJA0iN6X99+DfH2DhiHENjEmlbKN13+H+EPzm0K3ya2w8D6ANodB3AxMn2tGs8mLtG/P4kCjoDnoUVFzKTyUpTtgOjis47uSY4dL8mTbIK9kZAiFxnT5sMClu6KRStDjRIrxbTBQJ8wvlat admin1@rajula", "key_options": null, "keyfile": "/home/rpavani1998/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rpavani1998", "validate_certs": true} 2021-11-11T20:39:04.419 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rishabh', 'ovpn': 'rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rishabh", "ovpn": "rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4NYxUHx8HMgbVIHadruN1kAJS5be0aZA9rvGtYDfwD5siKNfXFRyLG3thkjxtEi7DPYMxgbpFOXW9EQKh5sQ0ohsADZOi8FSIAfQlOQ+7HAEp7DjBtjxoqB31jJ1e293YtO+nqAhcOMyGfDx+71w5sS2EFqy7PscA9gD9aV2iMxIR/JD1LxXabiqj12jU5AOxr7ZQ1+gM9aroQuDpNxY4U8jBTxx5OtIYRkeI/aN2gQj1mXGMKGo0ItFn4lsXGLxGxMpa9K2UFw2dWQVTw9VNm79nixrCX+lC2nyaJPsRYZipXF1ID2NRk7oJ79dJ/uRBBXpunihlEC3aF+nOgEPH ridave@redhat.com", "key_options": null, "keyfile": "/home/rishabh/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rishabh", "validate_certs": true} 2021-11-11T20:39:04.421 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rishabh', 'ovpn': 'rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rishabh", "ovpn": "rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4NYxUHx8HMgbVIHadruN1kAJS5be0aZA9rvGtYDfwD5siKNfXFRyLG3thkjxtEi7DPYMxgbpFOXW9EQKh5sQ0ohsADZOi8FSIAfQlOQ+7HAEp7DjBtjxoqB31jJ1e293YtO+nqAhcOMyGfDx+71w5sS2EFqy7PscA9gD9aV2iMxIR/JD1LxXabiqj12jU5AOxr7ZQ1+gM9aroQuDpNxY4U8jBTxx5OtIYRkeI/aN2gQj1mXGMKGo0ItFn4lsXGLxGxMpa9K2UFw2dWQVTw9VNm79nixrCX+lC2nyaJPsRYZipXF1ID2NRk7oJ79dJ/uRBBXpunihlEC3aF+nOgEPH ridave@redhat.com", "key_options": null, "keyfile": "/home/rishabh/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rishabh", "validate_certs": true} 2021-11-11T20:39:04.648 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'skrah', 'ovpn': 'skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "skrah", "ovpn": "skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCeAhoTum3/VmRbxSyw5gwgcPF1EWFK5mEPX3PZyo3xPsUAJazALfge3aEuCrZBbsjOCoj3GfFhr/ziEF+OWvhBoPK5hQzP0V8tE9cWJwvHmD72vGE36o1hFO0ONE4DEuu5KRDqQJUgeyQWXsP26uHTwIcvMpy8egZOfyshqF+lQVXzsAex7Ninmidsf/sRNLhzT1kLrKT+ZMSbZ7Rqh46TMlmLD1LzJZNcb6PGVwlfQZmV6T7m0COu/skeSxMcy8hqMvHFNvvfFSqkfMw0KBGLDxMQr4L+e7sXwQDnEZHdzoh0iS3xDp/Onr1FSmPyEjz51e1sMHBsqklHWidSZVLt/+giB4o6Wdgzc0um/QxW2Ne2qjtDlViOgRE9Fzcw/cRu/Xsn8qtXTxYyxXswjkA8qX7WR22OYgjYChw9siUsKkuD2yAbPMVl+cZWDeBhQnPMbDMVaghS4SAVt5GK16jij4+r3IE9vGHBmGwJYJ/OrtcxBrUrDopwsGLcU+WHdv0FXuiw+YbfnzpthC8rVj4oeJ2IUOQ5JB0ub7NmbR4QWjc33JfD+AeCB75x6io07V2riRjMD9gLC5nwytJNG7Yytf6xI2i/CprW7HhsDs2ykOVcDSBMnL1oGW6FeHaDRW39r6BXSt1jjZQ2E6pDEBTDO41tPEL9EVvv9TK87GTylQ==", "key_options": null, "keyfile": "/home/skrah/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "skrah", "validate_certs": true} 2021-11-11T20:39:04.659 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'skrah', 'ovpn': 'skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "skrah", "ovpn": "skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCeAhoTum3/VmRbxSyw5gwgcPF1EWFK5mEPX3PZyo3xPsUAJazALfge3aEuCrZBbsjOCoj3GfFhr/ziEF+OWvhBoPK5hQzP0V8tE9cWJwvHmD72vGE36o1hFO0ONE4DEuu5KRDqQJUgeyQWXsP26uHTwIcvMpy8egZOfyshqF+lQVXzsAex7Ninmidsf/sRNLhzT1kLrKT+ZMSbZ7Rqh46TMlmLD1LzJZNcb6PGVwlfQZmV6T7m0COu/skeSxMcy8hqMvHFNvvfFSqkfMw0KBGLDxMQr4L+e7sXwQDnEZHdzoh0iS3xDp/Onr1FSmPyEjz51e1sMHBsqklHWidSZVLt/+giB4o6Wdgzc0um/QxW2Ne2qjtDlViOgRE9Fzcw/cRu/Xsn8qtXTxYyxXswjkA8qX7WR22OYgjYChw9siUsKkuD2yAbPMVl+cZWDeBhQnPMbDMVaghS4SAVt5GK16jij4+r3IE9vGHBmGwJYJ/OrtcxBrUrDopwsGLcU+WHdv0FXuiw+YbfnzpthC8rVj4oeJ2IUOQ5JB0ub7NmbR4QWjc33JfD+AeCB75x6io07V2riRjMD9gLC5nwytJNG7Yytf6xI2i/CprW7HhsDs2ykOVcDSBMnL1oGW6FeHaDRW39r6BXSt1jjZQ2E6pDEBTDO41tPEL9EVvv9TK87GTylQ==", "key_options": null, "keyfile": "/home/skrah/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "skrah", "validate_certs": true} 2021-11-11T20:39:04.887 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'smanjara', 'ovpn': 'smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "smanjara", "ovpn": "smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDklgWHrL9TMaMxIyN9MxVR6nR3xh9zfj0iR63/yOzaslX0WoVBk7CYTsRlH2GFVctvixVwrWbQgMaKe49utal/3FA6sMl8S2DTrtgC1a/exUTvU0U/+ZhyUlkyjQfRgtYOUwodNupL5NsC8VZqZshgmgXC34uye3bplx3VwvpUt9r6QxJN3/9Dbi6HxudhDCLfmvROhwQYHaFF+GglcbyjAt3uvsuwXVKBd1/vMQwbcUo9/zQlrnmWBRuqTNkXGrZauWigoi/hyuWYQ3MUbRc3LVpJncMv7BEtcj8fCVvPB09SFee3nSeTtFkm01GvRcYd3HBz/YL4Rz92wXl9VIxqs8aaJBzy+MD0NO496Rfqtz6YuZ1m7AxDIccDW+OpcPgYwaUKFXNTuVLWZXhkS0p4swefcKWRlQFF3BLOebN/Kr9jq1MFa+5pX7Gj4WF5uCIX0/fx40KVZNhfl3qywBy/d5MKwXO7PCHMiGcu2FIXdZ334MxdgiIVoX54PpRP4hs= smanjara@localhost.localdomain", "key_options": null, "keyfile": "/home/smanjara/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "smanjara", "validate_certs": true} 2021-11-11T20:39:04.888 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'smanjara', 'ovpn': 'smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "smanjara", "ovpn": "smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDklgWHrL9TMaMxIyN9MxVR6nR3xh9zfj0iR63/yOzaslX0WoVBk7CYTsRlH2GFVctvixVwrWbQgMaKe49utal/3FA6sMl8S2DTrtgC1a/exUTvU0U/+ZhyUlkyjQfRgtYOUwodNupL5NsC8VZqZshgmgXC34uye3bplx3VwvpUt9r6QxJN3/9Dbi6HxudhDCLfmvROhwQYHaFF+GglcbyjAt3uvsuwXVKBd1/vMQwbcUo9/zQlrnmWBRuqTNkXGrZauWigoi/hyuWYQ3MUbRc3LVpJncMv7BEtcj8fCVvPB09SFee3nSeTtFkm01GvRcYd3HBz/YL4Rz92wXl9VIxqs8aaJBzy+MD0NO496Rfqtz6YuZ1m7AxDIccDW+OpcPgYwaUKFXNTuVLWZXhkS0p4swefcKWRlQFF3BLOebN/Kr9jq1MFa+5pX7Gj4WF5uCIX0/fx40KVZNhfl3qywBy/d5MKwXO7PCHMiGcu2FIXdZ334MxdgiIVoX54PpRP4hs= smanjara@localhost.localdomain", "key_options": null, "keyfile": "/home/smanjara/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "smanjara", "validate_certs": true} 2021-11-11T20:39:05.115 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'bengland', 'ovpn': 'bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "bengland", "ovpn": "bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDpp2NWr5T6s+jzHiz/FfCKzN7ZBnUSzyE7ieuntzjeGotyupAAjVBoSpCJgNDybm+WkE4dShv5mttPJ2/q1UZCLRj3nN+y3NG333aLZuyugR0ht5qtvGVegg58PwQ8CMobR7ca7bl5UdEy0UIrM3d/qpDOuPkuE4MIJcrtZGICE8NAVnmHNOB9Sm4ovAYkewTuMAv40j2Dl4F9dD6bg+OJfBOJ9+Zj+MpowYlAILtEB9Dc20zB8GGUEtr9yfOVroblTYhvTxGYRaE7aVBe83DNwkWGbDc/qUNR6bpA04w/P+LclYVsVKRoHOs1EXHSwNMFromA8RsKS+rajwu5uRg5 ben@dhcp-185-182.bos.redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC93wGgyyQmTxa7f+BZcYPUapTH2N1mnV1ikl9nzJzHhgrtJ6pv00LhtH1HqGhvoqnqZjBD5XRAKyjMRcMBT5MYcXXep5RqZ9sJ6Xz8rHkBjfl/OOQUeZ/ZyG/Ksr1jpGraN2xtrjQQzbqN1YvirCqfLk2exPvVeEk4CGnNybwKwCWob1uKV46ASrYeQK8pfHxrW7pTo+xdErHOvb7uumZm89/YisUJGicO1qSHBeuw1MxjhnD+fIcxHKWY5NNJ4qScW6a5AABk1JJsoHkh9IGCVaf/zJv8D3gil5SiM8LIBKaNa1fn6oObPl/Ky/wkiIo2gmUZawvAVA8iJnX9eUPl ben@bene-laptop", "key_options": null, "keyfile": "/home/bengland/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "bengland", "validate_certs": true} 2021-11-11T20:39:05.126 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'bengland', 'ovpn': 'bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "bengland", "ovpn": "bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDpp2NWr5T6s+jzHiz/FfCKzN7ZBnUSzyE7ieuntzjeGotyupAAjVBoSpCJgNDybm+WkE4dShv5mttPJ2/q1UZCLRj3nN+y3NG333aLZuyugR0ht5qtvGVegg58PwQ8CMobR7ca7bl5UdEy0UIrM3d/qpDOuPkuE4MIJcrtZGICE8NAVnmHNOB9Sm4ovAYkewTuMAv40j2Dl4F9dD6bg+OJfBOJ9+Zj+MpowYlAILtEB9Dc20zB8GGUEtr9yfOVroblTYhvTxGYRaE7aVBe83DNwkWGbDc/qUNR6bpA04w/P+LclYVsVKRoHOs1EXHSwNMFromA8RsKS+rajwu5uRg5 ben@dhcp-185-182.bos.redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC93wGgyyQmTxa7f+BZcYPUapTH2N1mnV1ikl9nzJzHhgrtJ6pv00LhtH1HqGhvoqnqZjBD5XRAKyjMRcMBT5MYcXXep5RqZ9sJ6Xz8rHkBjfl/OOQUeZ/ZyG/Ksr1jpGraN2xtrjQQzbqN1YvirCqfLk2exPvVeEk4CGnNybwKwCWob1uKV46ASrYeQK8pfHxrW7pTo+xdErHOvb7uumZm89/YisUJGicO1qSHBeuw1MxjhnD+fIcxHKWY5NNJ4qScW6a5AABk1JJsoHkh9IGCVaf/zJv8D3gil5SiM8LIBKaNa1fn6oObPl/Ky/wkiIo2gmUZawvAVA8iJnX9eUPl ben@bene-laptop", "key_options": null, "keyfile": "/home/bengland/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "bengland", "validate_certs": true} 2021-11-11T20:39:05.355 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'pnawracay', 'ovpn': 'pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pnawracay", "ovpn": "pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC3YhI4am/T6s4rJRvbC/jAvrNcDhE/zaJiVDAjEuA1W/5Xt9ObX52eAnp+tOOtelXrX8ylYF/z5wGNQLjYrWepXhuqaEGxeVayCHzts6LMpmvyZ6sYnAw9HFO9rzrTawm70DuvWqY2Zt4KA6fM+lfXk3Qdpi5RCPDKk1jpXwSCyUtUKSWTDRtYtvvJTcO3H/xCJxQq/F9sU8yqgQcqFPfMNrV4a8gNj9KACqjt4IEIfuxKlKhlMA4yu1VTmGcNuhQS+geJzsEGuxZUQb0Io+q4o0b+40BQtazKgOdTcQbgNtqh3wZpjJsvB4AM+GNPjZY3mCsd+WPGg6dqfU08Voo2b7BfSwyCn14fQHnj1nSwRIea2juFY6AMfVJjstM5iIz7lOH2c3ydWWhzvSmMeoQ47zBXlg8JytWmDrXiyd21BSIpTL0Tn8P95wHYka8U9KRV0ZY7Jr2w/ThPdy2C+Cdir+P+OEL1sKHSnmEyQEeONa9P5yH1Rd1kM/FNESAIRiMLGspzXQeWoJhHswVaKoSBW/7f4b9VENxda8qAQi6ob8A71cAjbpRrkUmd5EBCSFMNs7lscus0/deUVvlpsrMjDAJa8S9gqhGHNU0YvF1WBIz3paLRAHDm9DWrHc5lreizJzX5ul4P/Z4ITjpO+9ZxKOcyi8wnJ8Og+QUYLpqOIw==", "key_options": null, "keyfile": "/home/pnawracay/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pnawracay", "validate_certs": true} 2021-11-11T20:39:05.357 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'pnawracay', 'ovpn': 'pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pnawracay", "ovpn": "pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC3YhI4am/T6s4rJRvbC/jAvrNcDhE/zaJiVDAjEuA1W/5Xt9ObX52eAnp+tOOtelXrX8ylYF/z5wGNQLjYrWepXhuqaEGxeVayCHzts6LMpmvyZ6sYnAw9HFO9rzrTawm70DuvWqY2Zt4KA6fM+lfXk3Qdpi5RCPDKk1jpXwSCyUtUKSWTDRtYtvvJTcO3H/xCJxQq/F9sU8yqgQcqFPfMNrV4a8gNj9KACqjt4IEIfuxKlKhlMA4yu1VTmGcNuhQS+geJzsEGuxZUQb0Io+q4o0b+40BQtazKgOdTcQbgNtqh3wZpjJsvB4AM+GNPjZY3mCsd+WPGg6dqfU08Voo2b7BfSwyCn14fQHnj1nSwRIea2juFY6AMfVJjstM5iIz7lOH2c3ydWWhzvSmMeoQ47zBXlg8JytWmDrXiyd21BSIpTL0Tn8P95wHYka8U9KRV0ZY7Jr2w/ThPdy2C+Cdir+P+OEL1sKHSnmEyQEeONa9P5yH1Rd1kM/FNESAIRiMLGspzXQeWoJhHswVaKoSBW/7f4b9VENxda8qAQi6ob8A71cAjbpRrkUmd5EBCSFMNs7lscus0/deUVvlpsrMjDAJa8S9gqhGHNU0YvF1WBIz3paLRAHDm9DWrHc5lreizJzX5ul4P/Z4ITjpO+9ZxKOcyi8wnJ8Og+QUYLpqOIw==", "key_options": null, "keyfile": "/home/pnawracay/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pnawracay", "validate_certs": true} 2021-11-11T20:39:05.584 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'alfonsomthd', 'ovpn': 'alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "alfonsomthd", "ovpn": "alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQu/CuhCIFVGCLbA0cjn+qqcwX12cRcFA1sHyJtkj6Ltfmge8FnfgAsex/LaPJwbtbBiVl9xRcazHNfBaDw5DxMRSJF+e7WTydZC2Jm4qIj4D3Lr+imGPrJTK+kjN0QXRXknGsNn36oRF/I41cGqYRGwawFGSPZBHuav+ehDk1doTrbDlll4zMUWm2iXTcBIcJo4NR1LoE2c9/MPfbH7jKm5ke7aS0ets+EANEplkeClfCVAashtIF+Bz6a9qk4F2n0baLT/s3IQkXCwwO2fxxY03x8gWiB1n4E7sousEB+J1Skl/z0bwmaiY0C+raPoV/FyZxQuh0ocvdU7fnQbKuNoGE/Usfm9Y3g8eH81b/k4fMJL2lqXHkPft6zUV9bJsp3v1iGEkIf2+ER/QLczteNz6kw/uTPQctMEDXM17GEnBXISVrQcPSW3i93eFwWacs8iT3G4bjEQvpxSBQrhDdCCoxy8bVWqkVeDWw5u2q/3WL9Ov1NrY+G4/DYb2qONNQds0vTHCGvemU5GNjtaeCbojkZ7Z3KOeAY8080kKrsO7fve+ZMACCIFD+se8kF7NN4ZJASSqq9lJpBWvtbc0O8jCcVGNuN7exXjrcbiN1L/9HkARV+KEJU3jbXObAhAnfAaeRWBwmTfZodCu+fDd/Jor6l36tjD4kC4/CPrDtyQ== alfonsomthd", "key_options": null, "keyfile": "/home/alfonsomthd/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "alfonsomthd", "validate_certs": true} 2021-11-11T20:39:05.595 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'alfonsomthd', 'ovpn': 'alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "alfonsomthd", "ovpn": "alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQu/CuhCIFVGCLbA0cjn+qqcwX12cRcFA1sHyJtkj6Ltfmge8FnfgAsex/LaPJwbtbBiVl9xRcazHNfBaDw5DxMRSJF+e7WTydZC2Jm4qIj4D3Lr+imGPrJTK+kjN0QXRXknGsNn36oRF/I41cGqYRGwawFGSPZBHuav+ehDk1doTrbDlll4zMUWm2iXTcBIcJo4NR1LoE2c9/MPfbH7jKm5ke7aS0ets+EANEplkeClfCVAashtIF+Bz6a9qk4F2n0baLT/s3IQkXCwwO2fxxY03x8gWiB1n4E7sousEB+J1Skl/z0bwmaiY0C+raPoV/FyZxQuh0ocvdU7fnQbKuNoGE/Usfm9Y3g8eH81b/k4fMJL2lqXHkPft6zUV9bJsp3v1iGEkIf2+ER/QLczteNz6kw/uTPQctMEDXM17GEnBXISVrQcPSW3i93eFwWacs8iT3G4bjEQvpxSBQrhDdCCoxy8bVWqkVeDWw5u2q/3WL9Ov1NrY+G4/DYb2qONNQds0vTHCGvemU5GNjtaeCbojkZ7Z3KOeAY8080kKrsO7fve+ZMACCIFD+se8kF7NN4ZJASSqq9lJpBWvtbc0O8jCcVGNuN7exXjrcbiN1L/9HkARV+KEJU3jbXObAhAnfAaeRWBwmTfZodCu+fDd/Jor6l36tjD4kC4/CPrDtyQ== alfonsomthd", "key_options": null, "keyfile": "/home/alfonsomthd/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "alfonsomthd", "validate_certs": true} 2021-11-11T20:39:05.823 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'oliveiradan', 'ovpn': 'oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "oliveiradan", "ovpn": "oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCoL9+aMxLtUG3aRIuT4kbjc1CPuQ+wbS30ocuJ2YIExlKMSxVr+LYqbjcz0T4f8gAKG539dZQgD9BAsHM8yfOVEvei0KsQHn87pzgYGR3DtaMx5IEG5xNyARPu+VuEoe8qlL3cAPwTiaU78datD5Cz9GEClhL1Fhn/Z8o7BMbs5xlcUIQ/J+s9/sKMcqmmBJy5IHQIEUHCl4SFwsev5g5ADlnLNNxjpalExuBCEeoSma9F3b2YEJHvpA5JhCIMZpVfXbgh2UhUsyfAQN6KgOIQ/XlKBMQDtizf2vZ4b8hW4ELcvKSADfTsyyqEs/Ql8jEJu32GFKf2mYPbgTS6E7FZfQz77X+5OBhojFAstB8LSSrH80FrG1gCbVZLnW0jXcSehKnU0oNmUWOKxxaT1b00fo/iZ4P8O1uJRY17/uaVCPNKdGTZzIlzyC16wsFFBxBl0d7HfojQY6qfww7AQJhvDkV2sP0/MIJzwriWOzZEmfr8Osqery5vFiYcSvMbdMZDfMEYd3z7Va2RiTIeyXuEYp5hzf8eyJRP/f2FEw8/BBrm5JCxeSWUx7qwgPHrJoWAVxCIg/M3YS/vfVSFc8Hh88sgc2upq0OqYr0uq6EFzUZi4aoOkOVYn9u/uc/p1AR0KbiQOkERMUDWTyADxrHaL0nQmGy+XyiJQjhq5ckZ+w== doliveira@suse.com", "key_options": null, "keyfile": "/home/oliveiradan/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "oliveiradan", "validate_certs": true} 2021-11-11T20:39:05.825 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'oliveiradan', 'ovpn': 'oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "oliveiradan", "ovpn": "oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCoL9+aMxLtUG3aRIuT4kbjc1CPuQ+wbS30ocuJ2YIExlKMSxVr+LYqbjcz0T4f8gAKG539dZQgD9BAsHM8yfOVEvei0KsQHn87pzgYGR3DtaMx5IEG5xNyARPu+VuEoe8qlL3cAPwTiaU78datD5Cz9GEClhL1Fhn/Z8o7BMbs5xlcUIQ/J+s9/sKMcqmmBJy5IHQIEUHCl4SFwsev5g5ADlnLNNxjpalExuBCEeoSma9F3b2YEJHvpA5JhCIMZpVfXbgh2UhUsyfAQN6KgOIQ/XlKBMQDtizf2vZ4b8hW4ELcvKSADfTsyyqEs/Ql8jEJu32GFKf2mYPbgTS6E7FZfQz77X+5OBhojFAstB8LSSrH80FrG1gCbVZLnW0jXcSehKnU0oNmUWOKxxaT1b00fo/iZ4P8O1uJRY17/uaVCPNKdGTZzIlzyC16wsFFBxBl0d7HfojQY6qfww7AQJhvDkV2sP0/MIJzwriWOzZEmfr8Osqery5vFiYcSvMbdMZDfMEYd3z7Va2RiTIeyXuEYp5hzf8eyJRP/f2FEw8/BBrm5JCxeSWUx7qwgPHrJoWAVxCIg/M3YS/vfVSFc8Hh88sgc2upq0OqYr0uq6EFzUZi4aoOkOVYn9u/uc/p1AR0KbiQOkERMUDWTyADxrHaL0nQmGy+XyiJQjhq5ckZ+w== doliveira@suse.com", "key_options": null, "keyfile": "/home/oliveiradan/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "oliveiradan", "validate_certs": true} 2021-11-11T20:39:06.052 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'swagner', 'ovpn': 'swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "swagner", "ovpn": "swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDh9bzZJulXGES+l9Xh6Aq15RZ8uQCGuDNhlNQtDblE1ISKJ/DqGYXp6wUW54+oVNA7eZiXz+fi3mq5pPEtZZOfd3ixEDzDJ4E3cVXoDZCqeWEmea6KvybeY10YxvB56TEI8U2KKAd56PRl20klXpLCjzXNqG7n0aXFcpCMbXDu369VX6lOk24K/7++7Wc5SttcvVu19sT2kqzsB/S1Y5cxiE6RM6wVtqBoksp/kRCIA16ruNwx3GUabDfbEoUGXlkkUP7+TZgAbHtBsYy6mCQhwi0S2+WG+HhHDUPjHhV+MdN9ffibCtOEGo52itVLVky09VeBocuA6H22JDGXPBgjcgf2NsZIqcKqGHhUkXmH92fhRSFOBLKHstrBq8jWRP/mNrgj8cQksDsakQYQbDg5dyabp+M0/iL2Q3YVq7erZI8aZMA7ZF3WgoQNYZg5E7oejM8URIlFP3x1ne2ClRC9a74phSCxeU/NVamGN3G3dImzEXGOSNyRggHJ4jGrIGc7tLPCzmI5OkomcB5OxReqf0r1TNXuUAqw8M4EoWt+0xoAmH5zVlUHf+psUCJIEV/4pbgtoJiSNq+LVY4jyDEFbvTAL7MqyXorMV7Tqlj+/3d7RXfhW9lR/SHb1Z3jcHvz4ZzYMzKWeJEjz+Y0NwIdFDhcPmUOYtEDmuRRrgYiSQ== sebstian.wagner@it-novum.com", "key_options": null, "keyfile": "/home/swagner/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "swagner", "validate_certs": true} 2021-11-11T20:39:06.054 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'swagner', 'ovpn': 'swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "swagner", "ovpn": "swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDh9bzZJulXGES+l9Xh6Aq15RZ8uQCGuDNhlNQtDblE1ISKJ/DqGYXp6wUW54+oVNA7eZiXz+fi3mq5pPEtZZOfd3ixEDzDJ4E3cVXoDZCqeWEmea6KvybeY10YxvB56TEI8U2KKAd56PRl20klXpLCjzXNqG7n0aXFcpCMbXDu369VX6lOk24K/7++7Wc5SttcvVu19sT2kqzsB/S1Y5cxiE6RM6wVtqBoksp/kRCIA16ruNwx3GUabDfbEoUGXlkkUP7+TZgAbHtBsYy6mCQhwi0S2+WG+HhHDUPjHhV+MdN9ffibCtOEGo52itVLVky09VeBocuA6H22JDGXPBgjcgf2NsZIqcKqGHhUkXmH92fhRSFOBLKHstrBq8jWRP/mNrgj8cQksDsakQYQbDg5dyabp+M0/iL2Q3YVq7erZI8aZMA7ZF3WgoQNYZg5E7oejM8URIlFP3x1ne2ClRC9a74phSCxeU/NVamGN3G3dImzEXGOSNyRggHJ4jGrIGc7tLPCzmI5OkomcB5OxReqf0r1TNXuUAqw8M4EoWt+0xoAmH5zVlUHf+psUCJIEV/4pbgtoJiSNq+LVY4jyDEFbvTAL7MqyXorMV7Tqlj+/3d7RXfhW9lR/SHb1Z3jcHvz4ZzYMzKWeJEjz+Y0NwIdFDhcPmUOYtEDmuRRrgYiSQ== sebstian.wagner@it-novum.com", "key_options": null, "keyfile": "/home/swagner/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "swagner", "validate_certs": true} 2021-11-11T20:39:06.281 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'yuvalif', 'ovpn': 'ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "yuvalif", "ovpn": "ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDzYzqIDimPjMOUOUz1A122eV52fl3mgGgNBRYRlOWl/lOxM5oOat5FPkJlnXTVyQFl0MlnYgIv9fTnjHVTM0wFmFUItlUl151ZmK5XExiGpGVsqG3UWNyKJxZpk9Uh2a0T/5CKzzwXnv2fJMNxzfqgopYDrgq4TFEZcdvgWJ39dcRTXST/rGJQ8wOq9z7UNzKU5VOGxvgi6f/nGevGIHrSaTcY3Uu8BW8kXUrf9vfl+ZSxVjuNHXF/VmrJH5Ne8mVu7DzPMrl8ZZjxsfJKT/eQqF9kjqht5586OSPKIwg/LEOyYj7Wyyf3QbDAIp5bCygQfmBBl+kIxphJYq3EwUZT ylifshit@dhcp-4-102.tlv.redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDFgwQYD9amVSYq6JzjHyGmKBJATQCM9xsf6QnF3yb0Lvry2nzY9F61evVXQ95TmF/AqasRx5p2WansIvegG5ag5mytvRwFXtHzPs4o/J+XjA9o4+A5l6j9cOFLIDD/TwrwlUjgMa2PhqUztSpI1op906391RLrJAyflfDghJxeVl6dJab6TEsslV0W+yaCS3BBxMXXrCbxCB5Ms3KCRKDz0DqbhFc+Nw05L1O1pC98HBFRIp1SaXlMcT3++CRQ3wugpOMkDMNLbycl+vdMBFY3UZ28rRowS3Ta7V5EARZVVH7KxU5cL+8Hr/8DE1/ITOmrluaEZ1KwYgoxZbZ3w7Ptw0FK6aiCZ49YtqKT1WakoVfO8m7pCdxkqUOA91loMQtxvCwZ/Vfoxaut16gklmPz8N11xvo1H5XB8/ZrvK5yKdl3JDl+wwtfE/fwo4E/fte+rPRisoWBRsmsx8EGxs75sXnb6n3VjZ65vYEKsNMLg+5EAaevWLLeqF0a42UWGJrHdb4VKX/OAplHrOk7kA/VwArow8f9n+iaRp2hgDU9OhQiFVRhgqwahlmCFFXkcP4/4WlxMCuZrsaNMZgMjVhfsp7cskKOezSfZZey2MB2fREMS1wg0j40CdYRn4jTNEBr/LVszsoM7t7AwXUpnqXeYIe+gePx1PJb5XSzSPZ9Pw== yuvalif@yahoo.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDp0xz0ml2XuoQe70TxVeTo3Ib5xMinDrYNJ5eq5XYNa8ZLGcEByeDK4myzixTloAo31Rb/yDyGqv6kQ6nFChihO2aab3Ni/bsMOaJat1scrznaIYXGvLvw5KYx/qBDUxb3qW0vO0N/bB3dUPS1 2021-11-11T20:39:06.283 INFO:teuthology.task.ansible.out:lXTJaVf+KuljUk7v38ov2gsPWlofRYZNkPGQfUKXFEXc39OVj+Ex5LkehH/Z1LwjRYJa7jAdSm8hE5m8m6WborrzAA4/iFuLbhtM/Bqm4boSqa7koBVux422QX/HblSQ2KAeT1GFOS7xdcO6ndyp6mIpCZWunTjZwSLyK20NvIV/AzdAro4GZWi0D34PPtbSG344ZT1dSLeDRhYVkDjS6uutrmw9C+Ns1bo8s1ydhxL2XEsU14SWRWlbyrMF9PXyJOsaNiBNWGwc9lTly0ZlmPnN3l0AC3RIKOstkEYmI+TMj21r1tltz48e+6dH1an51wwnhNRYiODd3hbS67tmIg05BuSU5+whJ93wzXw2rC4FLE= ylifshit@localhost.localdomain", "key_options": null, "keyfile": "/home/yuvalif/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "yuvalif", "validate_certs": true} ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'yuvalif', 'ovpn': 'ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "yuvalif", "ovpn": "ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDzYzqIDimPjMOUOUz1A122eV52fl3mgGgNBRYRlOWl/lOxM5oOat5FPkJlnXTVyQFl0MlnYgIv9fTnjHVTM0wFmFUItlUl151ZmK5XExiGpGVsqG3UWNyKJxZpk9Uh2a0T/5CKzzwXnv2fJMNxzfqgopYDrgq4TFEZcdvgWJ39dcRTXST/rGJQ8wOq9z7UNzKU5VOGxvgi6f/nGevGIHrSaTcY3Uu8BW8kXUrf9vfl+ZSxVjuNHXF/VmrJH5Ne8mVu7DzPMrl8ZZjxsfJKT/eQqF9kjqht5586OSPKIwg/LEOyYj7Wyyf3QbDAIp5bCygQfmBBl+kIxphJYq3EwUZT ylifshit@dhcp-4-102.tlv.redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDFgwQYD9amVSYq6JzjHyGmKBJATQCM9xsf6QnF3yb0Lvry2nzY9F61evVXQ95TmF/AqasRx5p2WansIvegG5ag5mytvRwFXtHzPs4o/J+XjA9o4+A5l6j9cOFLIDD/TwrwlUjgMa2PhqUztSpI1op906391RLrJAyflfDghJxeVl6dJab6TEsslV0W+yaCS3BBxMXXrCbxCB5Ms3KCRKDz0DqbhFc+Nw05L1O1pC98HBFRIp1SaXlMcT3++CRQ3wugpOMkDMNLbycl+vdMBFY3UZ28rRowS3Ta7V5EARZVVH7KxU5cL+8Hr/8DE1/ITOmr 2021-11-11T20:39:06.285 INFO:teuthology.task.ansible.out:luaEZ1KwYgoxZbZ3w7Ptw0FK6aiCZ49YtqKT1WakoVfO8m7pCdxkqUOA91loMQtxvCwZ/Vfoxaut16gklmPz8N11xvo1H5XB8/ZrvK5yKdl3JDl+wwtfE/fwo4E/fte+rPRisoWBRsmsx8EGxs75sXnb6n3VjZ65vYEKsNMLg+5EAaevWLLeqF0a42UWGJrHdb4VKX/OAplHrOk7kA/VwArow8f9n+iaRp2hgDU9OhQiFVRhgqwahlmCFFXkcP4/4WlxMCuZrsaNMZgMjVhfsp7cskKOezSfZZey2MB2fREMS1wg0j40CdYRn4jTNEBr/LVszsoM7t7AwXUpnqXeYIe+gePx1PJb5XSzSPZ9Pw== yuvalif@yahoo.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDp0xz0ml2XuoQe70TxVeTo3Ib5xMinDrYNJ5eq5XYNa8ZLGcEByeDK4myzixTloAo31Rb/yDyGqv6kQ6nFChihO2aab3Ni/bsMOaJat1scrznaIYXGvLvw5KYx/qBDUxb3qW0vO0N/bB3dUPS1lXTJaVf+KuljUk7v38ov2gsPWlofRYZNkPGQfUKXFEXc39OVj+Ex5LkehH/Z1LwjRYJa7jAdSm8hE5m8m6WborrzAA4/iFuLbhtM/Bqm4boSqa7koBVux422QX/HblSQ2KAeT1GFOS7xdcO6ndyp6mIpCZWunTjZwSLyK20NvIV/AzdAro4GZWi0D34PPtbSG344ZT1dSLeDRhYVkDjS6uutrmw9C+Ns1bo8s1ydhxL2XEsU14SWRWlbyrMF9PXyJOsaNiBNWGwc9lTly0ZlmPnN3l0AC3RIKOstkEYmI+TMj21r1tltz48e+6dH1an51wwnhNRYiODd3hbS67tmIg05BuSU5+whJ93wzXw2rC4FLE= ylifshit@localhost.localdomain", "key_options": null, "keyfile": "/home/yuvalif/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "yuvalif", "validate_certs": true} 2021-11-11T20:39:06.510 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kkeithle', 'ovpn': 'kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kkeithle", "ovpn": "kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDXa9uTf3q+7tGnHzSxv0WbgpRKrBIYjAZ5x3V74vQUSD1aAbmyvOqgPnts7XttLYImjjRqiisyC03L3KgR4b3SOeyL3KFxrk3KkHAJ/AJH+Y/UafyDhHNl3gRNg0B9rjsa7h2dSlcrFKnAX0pDY1HapuNPrzYWrXsEbo7C6wteFh7PyPmTkHu0NzIIPC4FcLl8SGaww/HeBh8v91IymgFSgXG0jNC063FEExWdti54yNLGh3gpW0FytsiRN9eEOKUOgcveT1tejz9ORv8evO4Y9SCQ/lsyOKQKNcG5C7J8nsbwSjqQ2cf6i2aYFZ0Oq1UXuhJ8APJCoiKgLJy7AiYf kkeithle@dhcp-100-19-89.bos.redhat.com", "key_options": null, "keyfile": "/home/kkeithle/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kkeithle", "validate_certs": true} 2021-11-11T20:39:06.521 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kkeithle', 'ovpn': 'kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kkeithle", "ovpn": "kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDXa9uTf3q+7tGnHzSxv0WbgpRKrBIYjAZ5x3V74vQUSD1aAbmyvOqgPnts7XttLYImjjRqiisyC03L3KgR4b3SOeyL3KFxrk3KkHAJ/AJH+Y/UafyDhHNl3gRNg0B9rjsa7h2dSlcrFKnAX0pDY1HapuNPrzYWrXsEbo7C6wteFh7PyPmTkHu0NzIIPC4FcLl8SGaww/HeBh8v91IymgFSgXG0jNC063FEExWdti54yNLGh3gpW0FytsiRN9eEOKUOgcveT1tejz9ORv8evO4Y9SCQ/lsyOKQKNcG5C7J8nsbwSjqQ2cf6i2aYFZ0Oq1UXuhJ8APJCoiKgLJy7AiYf kkeithle@dhcp-100-19-89.bos.redhat.com", "key_options": null, "keyfile": "/home/kkeithle/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kkeithle", "validate_certs": true} 2021-11-11T20:39:06.741 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'emmericp', 'ovpn': 'paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "emmericp", "ovpn": "paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130"}, "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBYa5uEOlFv1s9K9D1SbcaApLwX3LBwASSnRXDHI7MtU paul.emmerich@croit.io", "key_options": null, "keyfile": "/home/emmericp/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "emmericp", "validate_certs": true} 2021-11-11T20:39:06.752 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'emmericp', 'ovpn': 'paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "emmericp", "ovpn": "paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130"}, "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBYa5uEOlFv1s9K9D1SbcaApLwX3LBwASSnRXDHI7MtU paul.emmerich@croit.io", "key_options": null, "keyfile": "/home/emmericp/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "emmericp", "validate_certs": true} 2021-11-11T20:39:06.981 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mchangir', 'ovpn': 'mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mchangir", "ovpn": "mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCj28V9sMybkAzhkiaNiDDr8TfNVyr5DNdbOqszdGEVLy1Rxh/UYnxUclsR9n8lt9VnCft3M4NDTnVwlxOHE8+0oolJE9HP+DQomsgj3abP04EGJyOA0X8E7eIFQNLRbMFCD0yX/SNNlYPsS5fPTFhIa/gbBPmDBE07kULts66Tvc0Yo/2yIfwV1DV34bcWWVGi01XybIHBl9qbvlfBKAUrFEcv3ZdpHNhuZw6IkZY8xoYxo8iF+toLYeGX+gFxqCZUUOO+NBVmb0hVL1LI6bU66IrMOQDkFurqpvoraBTJqwZgXvMPceFUkcVIH4Nm7cK1X2WyMtnSw2XBpLMT8C4V mchangir@indraprastha", "key_options": null, "keyfile": "/home/mchangir/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mchangir", "validate_certs": true} 2021-11-11T20:39:06.992 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mchangir', 'ovpn': 'mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mchangir", "ovpn": "mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCj28V9sMybkAzhkiaNiDDr8TfNVyr5DNdbOqszdGEVLy1Rxh/UYnxUclsR9n8lt9VnCft3M4NDTnVwlxOHE8+0oolJE9HP+DQomsgj3abP04EGJyOA0X8E7eIFQNLRbMFCD0yX/SNNlYPsS5fPTFhIa/gbBPmDBE07kULts66Tvc0Yo/2yIfwV1DV34bcWWVGi01XybIHBl9qbvlfBKAUrFEcv3ZdpHNhuZw6IkZY8xoYxo8iF+toLYeGX+gFxqCZUUOO+NBVmb0hVL1LI6bU66IrMOQDkFurqpvoraBTJqwZgXvMPceFUkcVIH4Nm7cK1X2WyMtnSw2XBpLMT8C4V mchangir@indraprastha", "key_options": null, "keyfile": "/home/mchangir/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mchangir", "validate_certs": true} 2021-11-11T20:39:07.220 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'sjust', 'ovpn': 'sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sjust", "ovpn": "sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6RDgqwA0GrvE6KqRaGCgYnFhnxdi9ehjIWvfaMa1oq0CqYxUAee2bNbp/X2Bd18Vqk8iH9fpE0+qrLgAH/8BpD+MtgGikCFMORvazeUtOCF/OCDO0AMg1jvl8aTdvrQU8ZTDPXa65Avpkc9vwugcoplnwrW7lKx++XqCWsSlfHgEdyMV3wrt/ZDFwI2AS2bZBpWM6ViG1KSjXjgTi/T3hSZPY+buYvWjzoEjQO0946iI3iiKKCGohEpCUnG02oJXcS0vvgl8dWE71451lbainEKJ4hbDhfDmy57s6VXfVUANsczwiPWiSC/ZmMlDaCCBeKHO0Gz0xXaDXd9j4SA7P sam@Pondermatic\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCYylAUS12hF6XndDQ3CkrUddKmXNSd8pOlHwO+pWaDTJPKxwtPL6sM3aOtaAPLrGkv1zD2k3iZaxmDwfPYGaXOyb0KS5ZCddtOyNXj5JLrjxh2niqFPP8SIkdHLOP+0Hv1/tB0Jz75RV+tNIT1SJTm7OZ8wZNy0qYycITSooJ14iEgyZ7rOa3EEzbtIVpHS4+YqIuIJdV2oV5egCrJyRv4sPUrS8ERXD4QZSOlHE+W4Tf+WVrEfsHOdDLRlAXJUzhuqgWtVouiYcW5ex294z28eA+12lKaPQpE0cft/NJ6FLs41yqh4ZN588bl8Ada+yTXEaHWioQEEJMlca2jIgrP sjust@rex004.front.sepia.ceph.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC08RrDQ2M7Zjcpx7RL9e+tjv48kBgDNyTenX1QdxW22JA2JR3D3ThA/ej9HtiPWcIVz16oZ4tzrCEVwaCMfbsmXq 2021-11-11T20:39:07.222 INFO:teuthology.task.ansible.out:2z08ZXYqwiBuoXJVshU7lIKVqAtadT4F6Q9eQvxEo4zWdBib1W0/cqNLlYzU/DuwlUqYnWBDLGxc4fD5Kxrz366oWUDB/ULCpl77TMqfTKlStCgGXot0zLqiscqzq1W7XsuA7A5vQw/JTq7MvIME+SkUuGWX0WKwTMIRZKNWBGmlf51bNru3GmmAnKBUFsJv9H4G0FPqGY/aH02rr5dAgTTmfKt+XbWRfMTMpjJqVRGX03+1mQNoYQ3kNDz0wl sjust@office\nssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5UMPySf/h9r2/OryRXrO9Qq9k5J9Dn9nvJYyaYCE82H/qkBJGboAJgyIXfbsGYO5uPM25d3fGRH4magzAQm2hGWNbbt1YWn9HaIyKCCXo9JX600/dv3OZbHyGGmgXPa4Anl16j1c2vxRRt2+ArjlyyhP7cAcRkilUGdagK9u6ALrwnz9hhXYV+BmXrxMm7zZCmdWHcWz+YxoA65es4+w+aLvVZWmUCZLF8uZrcpPSw+oK3SfZE8JPRc5vWulXbWEEfIoHb9AszPIpSyzbfPWXelmc3XlG0KD66t7w+AnNlQPt3JVePRCfFHPYeTj9B0k8sxPyUqTQIl5pu6GppRMOQ== sam@Deepthought", "key_options": null, "keyfile": "/home/sjust/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sjust", "validate_certs": true} ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'sjust', 'ovpn': 'sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sjust", "ovpn": "sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6RDgqwA0GrvE6KqRaGCgYnFhnxdi9ehjIWvfaMa1oq0CqYxUAee2bNbp/X2Bd18Vqk8iH 2021-11-11T20:39:07.223 INFO:teuthology.task.ansible.out:9fpE0+qrLgAH/8BpD+MtgGikCFMORvazeUtOCF/OCDO0AMg1jvl8aTdvrQU8ZTDPXa65Avpkc9vwugcoplnwrW7lKx++XqCWsSlfHgEdyMV3wrt/ZDFwI2AS2bZBpWM6ViG1KSjXjgTi/T3hSZPY+buYvWjzoEjQO0946iI3iiKKCGohEpCUnG02oJXcS0vvgl8dWE71451lbainEKJ4hbDhfDmy57s6VXfVUANsczwiPWiSC/ZmMlDaCCBeKHO0Gz0xXaDXd9j4SA7P sam@Pondermatic\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCYylAUS12hF6XndDQ3CkrUddKmXNSd8pOlHwO+pWaDTJPKxwtPL6sM3aOtaAPLrGkv1zD2k3iZaxmDwfPYGaXOyb0KS5ZCddtOyNXj5JLrjxh2niqFPP8SIkdHLOP+0Hv1/tB0Jz75RV+tNIT1SJTm7OZ8wZNy0qYycITSooJ14iEgyZ7rOa3EEzbtIVpHS4+YqIuIJdV2oV5egCrJyRv4sPUrS8ERXD4QZSOlHE+W4Tf+WVrEfsHOdDLRlAXJUzhuqgWtVouiYcW5ex294z28eA+12lKaPQpE0cft/NJ6FLs41yqh4ZN588bl8Ada+yTXEaHWioQEEJMlca2jIgrP sjust@rex004.front.sepia.ceph.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC08RrDQ2M7Zjcpx7RL9e+tjv48kBgDNyTenX1QdxW22JA2JR3D3ThA/ej9HtiPWcIVz16oZ4tzrCEVwaCMfbsmXq2z08ZXYqwiBuoXJVshU7lIKVqAtadT4F6Q9eQvxEo4zWdBib1W0/cqNLlYzU/DuwlUqYnWBDLGxc4fD5Kxrz366oWUDB/ULCpl77TMqfTKlStCgGXot0zLqiscqzq1W7XsuA7A5vQw/JTq7MvIME+SkUuGWX0WKwTMIRZKNWBGmlf51bNru3GmmAnKBUFsJv9H4G0FPqGY/aH02rr5dAgTTmfKt+XbWRfMTMpjJqVRGX03+1mQNoYQ3kNDz0wl sjust@office\nssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5UMPySf/h9r2/OryRXrO9Qq9k5J9Dn9nvJYyaYCE82H/qkBJGboAJgyIXfbsGYO5uPM25d3fGRH4magzAQm2hGWNbbt1YWn9HaIyKCCXo9JX600/dv3OZbHyGGmgXPa4Anl16j1c2vxRRt2+ArjlyyhP7cAcRkilUGdagK9u6ALrwnz9hhXYV+BmXrxMm7zZCmdWHcWz+YxoA65es4+w+aLvVZWmUCZLF8uZrcpPSw+oK3SfZE8JPRc5vWulXbWEEfIoHb9AszPIpSyzbfPWXelmc3XlG0KD66t7w+AnNlQPt3JVePRCfFHPYeTj9B0k8sxPyUqTQIl5pu6GppRMOQ== sam@Deepthought", "key_options": null, "keyfile": "/home/sjust/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sjust", "validate_certs": true} 2021-11-11T20:39:07.449 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ideepika', 'ovpn': 'deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ideepika", "ovpn": "deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDj6DxTZNodVnwJZrnSZ/jv9Cw4uYLb8MGpccmOvtwl93sVBvqq597AYnBqYiV4VBbKU3GVFbT41zkplia1iCfXoddttCOyWmEKviBXRnAsd5f80dxL0pfzrBJvCaH4NAfYgVw9JFWsvT981b+nB6vum4Vz8Dw0PM5GPnVUjsZNFkpmDPUBL4yHPmMCGP2WtUurJVD3Gm9t72XlG6Dl/vB9Qw/i4IY1SoCCbuhhNgE7IN8JXeGmVJbRwDeUlP+Q8Ti7iedV7YB/W1opt6eT5EPzX8NcxTc0c7rrDgsUWjFQYRSup0GLK9ofdHs0E2RbPPO5OSc5Z2+B1IEV7fZeyJwAyHwUgwx7sSGy7aROdvinaj/hOBglcgISyQ2CY7LTV8NGWJquo7yRCJXhZVIHfgqfBcrgWdNbr+vNBXnWe5PCtLhV8ryHPSJ/7Ru+7voQxgvwux5SdZwxXTbWDKni1f6/8bNrhRx6bmLK9xuYAgt2FhlIivI7L5kQF4etcBiaf68= deepika@x1cabon\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCdqbYcbJYtgCrAbLo7PcS82caXxcT8cpX0RkULaFXfaiUDs8Foy6Yf/XoEHIdpgw8kX6qBdydYXxti6HFhYDNsL5KleKu/yA+bVKlvri39CRNkNxFHAUL2CRBT5Hefj1AJQBm/BKVEN4/eb05OlJJ808Qb/aNycot/YwgpNvUIaO2yTMA0meq82wNdZmUcmyLB9QXz4QInTcoAsxq70KxX95YC938rTWpbg1hM8gzLkJJt/OD/Z7TMYv/Sc5KGOFqo930AmObQ5KPmffkWGYbLT65TxlCULnvIr63lWHdGTj5mYjib/v9ncu24az9GYRWYcicV0B/def+KVrrIp3Mz ideepika@teuthology", "key_options": null, "keyfile": "/home/ideepika/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ideepika", "validate_certs": true} 2021-11-11T20:39:07.451 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ideepika', 'ovpn': 'deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ideepika", "ovpn": "deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDj6DxTZNodVnwJZrnSZ/jv9Cw4uYLb8MGpccmOvtwl93sVBvqq597AYnBqYiV4VBbKU3GVFbT41zkplia1iCfXoddttCOyWmEKviBXRnAsd5f80dxL0pfzrBJvCaH4NAfYgVw9JFWsvT981b+nB6vum4Vz8Dw0PM5GPnVUjsZNFkpmDPUBL4yHPmMCGP2WtUurJVD3Gm9t72XlG6Dl/vB9Qw/i4IY1SoCCbuhhNgE7IN8JXeGmVJbRwDeUlP+Q8Ti7iedV7YB/W1opt6eT5EPzX8NcxTc0c7rrDgsUWjFQYRSup0GLK9ofdHs0E2RbPPO5OSc5Z2+B1IEV7fZeyJwAyHwUgwx7sSGy7aROdvinaj/hOBglcgISyQ2CY7LTV8NGWJquo7yRCJXhZVIHfgqfBcrgWdNbr+vNBXnWe5PCtLhV8ryHPSJ/7Ru+7voQxgvwux5SdZwxXTbWDKni1f6/8bNrhRx6bmLK9xuYAgt2FhlIivI7L5kQF4etcBiaf68= deepika@x1cabon\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCdqbYcbJYtgCrAbLo7PcS82caXxcT8cpX0RkULaFXfaiUDs8Foy6Yf/XoEHIdpgw8kX6qBdydYXxti6HFhYDNsL5KleKu/yA+bVKlvri39CRNkNxFHAUL2CRBT5Hefj1AJQBm/BKVEN4/eb05OlJJ808Qb/aNycot/YwgpNvUIaO2yTMA0meq82wNdZmUcmyLB9QXz4QInTcoAsxq70KxX95YC938rTWpbg1hM8gzLkJJt/OD/Z7TMYv/Sc5KGOFqo930AmObQ5KPmffkWGYbLT65TxlCULnvIr63lWHdGTj5mYjib/v9ncu24az9GYRWYcicV0B/def+KVrrIp3Mz ideepika@teuthology", "key_options": null, "keyfile": "/home/ideepika/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ideepika", "validate_certs": true} 2021-11-11T20:39:07.678 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'gsalomon', 'ovpn': 'gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "gsalomon", "ovpn": "gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDC0h+TpaUkm2V6Hbf7aPCTys/nsm51CmY9s+bdwEXAiSAWpq/7Xi2LlN0mZgsrAYBHbr5L+Pz8x0YdigvWysUK6U9XRSPM28cbWmh8F25re9HEk8ARESkMZKTv/hKFs1v7kTHbT9hNNynbQGQOqEaazjr0shcli+Byvop27riX3QDDSsd5xfCjjyDz9fiA9YESlF0kbXyFUKPsoaBhRv9FUhtH6DxlygxtxTsKyKgeoiKw2sCfeUMckcJKnGCMtkXW4U8h7NL/fPZFyBaMDRi/TtoBKaVGLaWqF7RuukfjoMCwhuIDmuTG3WhanH4HJrfNenPZkpZUBhzWwZcleao8SD3mXXfZ9gcZb8QB2ymEtTqrAEyDoNWO4q8ljNP3WJXIgF0h0Sjmx/CV/i0ArEYlBMwBD7DFVnPJqCqSyypuQdDGsDNQZxaRCiYg7nIslFOpDr+0PLp78m+CHyYbSFLTvVVJoJ1k0YhMnDPHM29nt7fMGx6Qrigwnuk0yKg1bfgJ/t0TOqEw0bqGKIvHnyoP/n0ObazEsiPl5pRr2ehs/Rqf35Iwhc1i02Hhz/3Sqwc1eAgqm4/OVghxqUJGpRocua5Gf/vO32wwoyGxh2sl7y6K2pNDA3glhKYbZeIh1fjnaTjikgleqIQkCBxbTeAE7n5+T6TcFtnII6RSxYv3tQ== gal.salomon@gmail.com", "key_options": null, "keyfile": "/home/gsalomon/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "gsalomon", "validate_certs": true} 2021-11-11T20:39:07.680 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'gsalomon', 'ovpn': 'gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "gsalomon", "ovpn": "gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDC0h+TpaUkm2V6Hbf7aPCTys/nsm51CmY9s+bdwEXAiSAWpq/7Xi2LlN0mZgsrAYBHbr5L+Pz8x0YdigvWysUK6U9XRSPM28cbWmh8F25re9HEk8ARESkMZKTv/hKFs1v7kTHbT9hNNynbQGQOqEaazjr0shcli+Byvop27riX3QDDSsd5xfCjjyDz9fiA9YESlF0kbXyFUKPsoaBhRv9FUhtH6DxlygxtxTsKyKgeoiKw2sCfeUMckcJKnGCMtkXW4U8h7NL/fPZFyBaMDRi/TtoBKaVGLaWqF7RuukfjoMCwhuIDmuTG3WhanH4HJrfNenPZkpZUBhzWwZcleao8SD3mXXfZ9gcZb8QB2ymEtTqrAEyDoNWO4q8ljNP3WJXIgF0h0Sjmx/CV/i0ArEYlBMwBD7DFVnPJqCqSyypuQdDGsDNQZxaRCiYg7nIslFOpDr+0PLp78m+CHyYbSFLTvVVJoJ1k0YhMnDPHM29nt7fMGx6Qrigwnuk0yKg1bfgJ/t0TOqEw0bqGKIvHnyoP/n0ObazEsiPl5pRr2ehs/Rqf35Iwhc1i02Hhz/3Sqwc1eAgqm4/OVghxqUJGpRocua5Gf/vO32wwoyGxh2sl7y6K2pNDA3glhKYbZeIh1fjnaTjikgleqIQkCBxbTeAE7n5+T6TcFtnII6RSxYv3tQ== gal.salomon@gmail.com", "key_options": null, "keyfile": "/home/gsalomon/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "gsalomon", "validate_certs": true} 2021-11-11T20:39:07.907 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'soumyakoduri', 'ovpn': 'skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "soumyakoduri", "ovpn": "skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCrGYBGPDQqn+2jnxKhDurSrAJvTT8dZMYGVCt6NbjpFhDFYtyvrF8Jv5NKBQynG18LwscN7a57L0Oq0e4jOEPRz//CDcAfP+L6p45L4BdUzTysN4+rQ7waMnIiYp7VjSswhGH/BnYIVKwmDqzM2fo1w/oLIT6mgwUYqAdceeswAkXBt7jgagL8ox3E4oHBqER7f60AHdTviuW953t5luVQiMjyF2fOwZuo2ycufIzBee0CwS+xyRuJQwdgD3HL4Qs2vtBCX+H8n3wC6A9JrMA5tzCDr5VZKBJZURIniwvaXfJodjO4SvBL5jAfXoMqriv8ZWHbap2fk82YE8/uPqNvSISjixJEBgQinklZ+iAOYm5tfPbhPM/8ErpN/yJ0EoGRn+nj6X6nAjXAIk3AENopeqtM7mLZ26RFIuEV5ld7eqUvAfiy6Hk2LH9Q1Z4CXaxD0gDGzUxgdAaaAgLAR4btWi2n7Tb7ZNDdOfctg2ya9ftzZBOHvCaVXSpJCry/GMU= skoduri@localhost.localdomain", "key_options": null, "keyfile": "/home/soumyakoduri/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "soumyakoduri", "validate_certs": true} 2021-11-11T20:39:07.909 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'soumyakoduri', 'ovpn': 'skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "soumyakoduri", "ovpn": "skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCrGYBGPDQqn+2jnxKhDurSrAJvTT8dZMYGVCt6NbjpFhDFYtyvrF8Jv5NKBQynG18LwscN7a57L0Oq0e4jOEPRz//CDcAfP+L6p45L4BdUzTysN4+rQ7waMnIiYp7VjSswhGH/BnYIVKwmDqzM2fo1w/oLIT6mgwUYqAdceeswAkXBt7jgagL8ox3E4oHBqER7f60AHdTviuW953t5luVQiMjyF2fOwZuo2ycufIzBee0CwS+xyRuJQwdgD3HL4Qs2vtBCX+H8n3wC6A9JrMA5tzCDr5VZKBJZURIniwvaXfJodjO4SvBL5jAfXoMqriv8ZWHbap2fk82YE8/uPqNvSISjixJEBgQinklZ+iAOYm5tfPbhPM/8ErpN/yJ0EoGRn+nj6X6nAjXAIk3AENopeqtM7mLZ26RFIuEV5ld7eqUvAfiy6Hk2LH9Q1Z4CXaxD0gDGzUxgdAaaAgLAR4btWi2n7Tb7ZNDdOfctg2ya9ftzZBOHvCaVXSpJCry/GMU= skoduri@localhost.localdomain", "key_options": null, "keyfile": "/home/soumyakoduri/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "soumyakoduri", "validate_certs": true} 2021-11-11T20:39:08.137 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kyr', 'ovpn': 'kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kyr", "ovpn": "kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPeuSTD7kv8yQX6dW+7Vpq1mqrcr8JGubPYa1DpGTV/gE50d5L42UspUN2FXmICDMiV1h4Q9WQAFma3168f1I5I94qFkV+/56szCMoILClm7IW/EV4t8fTxoyOpEa9l4qSU3qgJpDbY04CkiyVJFuQj24Zq3gD/70jYgGpy8RfA0BihSR6F+A0WZLh/fESjqfor3/pFWhUCRnPEqrZsXyuB8lRgDTyGp4s1cHyUqmFMuCPDRMtok0rSVpaRZj2Zj8lboah19UI/L89ptQgEdqsOKzlkxiT+1GdyIY+jpPCM0a3yZz75l3Qiu0k6q7qUV8T0hm1GZiBguvST1KMcwnt kyrylo.shatskyy@suse.com", "key_options": null, "keyfile": "/home/kyr/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kyr", "validate_certs": true} 2021-11-11T20:39:08.139 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kyr', 'ovpn': 'kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kyr", "ovpn": "kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPeuSTD7kv8yQX6dW+7Vpq1mqrcr8JGubPYa1DpGTV/gE50d5L42UspUN2FXmICDMiV1h4Q9WQAFma3168f1I5I94qFkV+/56szCMoILClm7IW/EV4t8fTxoyOpEa9l4qSU3qgJpDbY04CkiyVJFuQj24Zq3gD/70jYgGpy8RfA0BihSR6F+A0WZLh/fESjqfor3/pFWhUCRnPEqrZsXyuB8lRgDTyGp4s1cHyUqmFMuCPDRMtok0rSVpaRZj2Zj8lboah19UI/L89ptQgEdqsOKzlkxiT+1GdyIY+jpPCM0a3yZz75l3Qiu0k6q7qUV8T0hm1GZiBguvST1KMcwnt kyrylo.shatskyy@suse.com", "key_options": null, "keyfile": "/home/kyr/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kyr", "validate_certs": true} 2021-11-11T20:39:08.366 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'sseshasa', 'ovpn': 'sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sseshasa", "ovpn": "sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDeERrQ0NVXhuTYzI08rlKRbHZVFXSv1YC7sGq6aCWRbTCv7Ad/t/3OJz3vfcd2TjBMvABfrYCFgTFWIaJBaeA3BXUUU9y9TwnzuxKv4C46WJYoalVGyLEpWpcGbJ5v+leA6Je/+lunenplb7TBKGE/YTgggEXyX7KnunYlv+Jdm33TmAZTcHrLk5QR/6FdRGKmdlLna0dXZc3v04+AI8W+p9ghAv1+e8ENxJDTF8p0MawBiKWcSj2mysOhQ2Cj91qZ0qTnLLeu0CPf4rBO+qjYIshiCuTLhYt2UazlzME4nCCsVqK7Nz9olA/lgvJLRaPxgQQ7iCMmIZilS6B4Y3oL sseshasa@localhost.localdomain\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCo0zniUEXuQ4JJuuRtfnEK8WHqr97AmXEniGPY5U23MTxhMuP8CSkGWkUHdmXw0H1tDKbYgk97M6TspJropNGKljla2H1/3e2T1BOI1L2WfcsOh6VPYeyOj2ssMJeEDdQgd0J3g+4oj2QS+guPlT24UhO0CBhRZ6waVlJMgagDYyDLCLGI4ASoBYx0AOaoyjnEQtRd/gyejzv7jntkHc9YNBPsJ37X6hTYNA/s2ORJE7QMXd6xI60mTBuCdD3JcPYoUSvyCUa7rIrDc70lAcW5XtxewvMBogTZJLoFG/d6Ugij49h4AfwrtoUCtLyJOWJX6PTfomXely8Ym6KpN/3981+m70dhcHPDJ66QCWMd1LAgftVXTIsebA+Bxfg4Vsbrv7j1lJXzmMfsOl+1wVPe+Y9h28GyeIlalW3laURZGV7dhexGZ8YNxPuzxtAlWtfw+kddbjw0j4ODtgVXrj4SDxqmTTYYuMBkrQ3H33sOgPLx8DyA6e4pckb6lOOEvb41Yi5tyP7BIJlYN2PpNVDkSQd/t5Q0di4Ct+3vqLRCkPuJiJAUyaBGT4HM7yQc25+Y0B3sXNON7BjeaBXB7OwKZMp9w1jwbJt4JL/aGOw3HVljVyVgu+pIuQ2ISd9VBcmAJK5EGdWe5Q57h5fJkD+TVrXgeNTaufdclh5vWOH1qQ== sseshasa@teuthology", "key_options": null, "keyfile": "/home/sseshasa/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sseshasa", "validate_certs": true} 2021-11-11T20:39:08.368 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'sseshasa', 'ovpn': 'sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sseshasa", "ovpn": "sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDeERrQ0NVXhuTYzI08rlKRbHZVFXSv1YC7sGq6aCWRbTCv7Ad/t/3OJz3vfcd2TjBMvABfrYCFgTFWIaJBaeA3BXUUU9y9TwnzuxKv4C46WJYoalVGyLEpWpcGbJ5v+leA6Je/+lunenplb7TBKGE/YTgggEXyX7KnunYlv+Jdm33TmAZTcHrLk5QR/6FdRGKmdlLna0dXZc3v04+AI8W+p9ghAv1+e8ENxJDTF8p0MawBiKWcSj2mysOhQ2Cj91qZ0qTnLLeu0CPf4rBO+qjYIshiCuTLhYt2UazlzME4nCCsVqK7Nz9olA/lgvJLRaPxgQQ7iCMmIZilS6B4Y3oL sseshasa@localhost.localdomain\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCo0zniUEXuQ4JJuuRtfnEK8WHqr97AmXEniGPY5U23MTxhMuP8CSkGWkUHdmXw0H1tDKbYgk97M6TspJropNGKljla2H1/3e2T1BOI1L2WfcsOh6VPYeyOj2ssMJeEDdQgd0J3g+4oj2QS+guPlT24UhO0CBhRZ6waVlJMgagDYyDLCLGI4ASoBYx0AOaoyjnEQtRd/gyejzv7jntkHc9YNBPsJ37X6hTYNA/s2ORJE7QMXd6xI60mTBuCdD3JcPYoUSvyCUa7rIrDc70lAcW5XtxewvMBogTZJLoFG/d6Ugij49h4AfwrtoUCtLyJOWJX6PTfomXely8Ym6KpN/3981+m70dhcHPDJ66QCWMd1LAgftVXTIsebA+Bxfg4Vsbrv7j1lJXzmMfsOl+1wVPe+Y9h28GyeIlalW3laURZGV7dhexGZ8YNxPuzxtAlWtfw+kddbjw0j4ODtgVXrj4SDxqmTTYYuMBkrQ3H33sOgPLx8DyA6e4pckb6lOOEvb41Yi5tyP7BIJlYN2PpNVDkSQd/t5Q0di4Ct+3vqLRCkPuJiJAUyaBGT4HM7yQc25+Y0B3sXNON7BjeaBXB7OwKZMp9w1jwbJt4JL/aGOw3HVljVyVgu+pIuQ2ISd9VBcmAJK5EGdWe5Q57h5fJkD+TVrXgeNTaufdclh5vWOH1qQ== sseshasa@teuthology", "key_options": null, "keyfile": "/home/sseshasa/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sseshasa", "validate_certs": true} 2021-11-11T20:39:08.594 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rfriedma', 'ovpn': 'rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rfriedma", "ovpn": "rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDIC6uqbIg9XaTlDZd7KCX7FXSzfGFMEWXhx42ktz3ehOPLviA/yNpVd+LmYHSgzd5Szd3mMxleisiE+JdSq/q5YyPAF6eD6+wfynkpMv5aWNHGI0yVfKbCwdOX8bY6yqhyVwMDo14uoFvdDcM4Lj+vz8F75VJHUU5ihJNOW24EllobTLzR4XI6AcM+mjbXRnBLLVoBSuDaU2D0kWqMn8rDCJKdUJnoiy9oxIok8eMLfGkGsL55oOx9BflMSz/0jM9X5GLQahYiECbXjvCIarCkUFE8txNoXM9Djmg8QqhAQDhY0hzJTauFJJaD8Ax1uFQlXONed7tv7a7Tb3dH+/jb rfriedma@dhcp-4-239.tlv.redhat.com", "key_options": null, "keyfile": "/home/rfriedma/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rfriedma", "validate_certs": true} 2021-11-11T20:39:08.605 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rfriedma', 'ovpn': 'rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rfriedma", "ovpn": "rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDIC6uqbIg9XaTlDZd7KCX7FXSzfGFMEWXhx42ktz3ehOPLviA/yNpVd+LmYHSgzd5Szd3mMxleisiE+JdSq/q5YyPAF6eD6+wfynkpMv5aWNHGI0yVfKbCwdOX8bY6yqhyVwMDo14uoFvdDcM4Lj+vz8F75VJHUU5ihJNOW24EllobTLzR4XI6AcM+mjbXRnBLLVoBSuDaU2D0kWqMn8rDCJKdUJnoiy9oxIok8eMLfGkGsL55oOx9BflMSz/0jM9X5GLQahYiECbXjvCIarCkUFE8txNoXM9Djmg8QqhAQDhY0hzJTauFJJaD8Ax1uFQlXONed7tv7a7Tb3dH+/jb rfriedma@dhcp-4-239.tlv.redhat.com", "key_options": null, "keyfile": "/home/rfriedma/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rfriedma", "validate_certs": true} 2021-11-11T20:39:08.834 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'lmb', 'ovpn': 'lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "lmb", "ovpn": "lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDIXHA8BwV4OnqlaHHyYofsp5X+wanose+aneJVx2Or05PcWA9c7xFEOerkvEn9LvivSRd9G9o/S8SvnGH91tG2bbOAidVZ69FGul9OKa/BCJYUP0vVBX09/Td3n7+R2lVlvu5XiyC5SUzCjaQEZLOpjq23rqaqdCDXE6wFnxfpkmcsccbG/xUiOEBeBGI92lntdx99Sn6CxWWA27lMMaYkCRvX6ziRZEm6O44N2nMm+qYm0jmMyGv1iyQkJxtt0YaLPp+X9t2trUEzEjgBCXqqzEzBjB7T4rbzhAuvU3NRHiAoWXSnrY5AQOTTvC/wpaZV/gF2NDNThjMcpcEdNstleOctXtN4XTTJQewbG+t7I9knR00TPpsoUvUtpbcoFOluGgWTDTFa6GCmDPppx23nZgulOniZUVTI2CbScSpc8sfp1JE7Q6vLECY9lJ+GpVCbDAbIEE9WHXSkJ/6L9yf+ZoHbpqCkEFDcWLfz3B5NFoJ5gne8B8uzd6ptKVtJrP8pMvoOyydyvQlmkgh8tNDsOnp6kaUNG0NHmtfGlCTkbM7oKsKbf87Cv1HgYuZl/HP7go5sOX02LfBfbvWrawZzOaZ3dewEq74w5hVMKEHrmJPstIpcHSxZpq33f2j4Z/uKW9tJat6rgSWK1I+T3tCd++76apxbOYl5TDOaoxCyjQ== lars@marowsky-bree.de", "key_options": null, "keyfile": "/home/lmb/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "lmb", "validate_certs": true} 2021-11-11T20:39:08.836 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'lmb', 'ovpn': 'lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "lmb", "ovpn": "lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDIXHA8BwV4OnqlaHHyYofsp5X+wanose+aneJVx2Or05PcWA9c7xFEOerkvEn9LvivSRd9G9o/S8SvnGH91tG2bbOAidVZ69FGul9OKa/BCJYUP0vVBX09/Td3n7+R2lVlvu5XiyC5SUzCjaQEZLOpjq23rqaqdCDXE6wFnxfpkmcsccbG/xUiOEBeBGI92lntdx99Sn6CxWWA27lMMaYkCRvX6ziRZEm6O44N2nMm+qYm0jmMyGv1iyQkJxtt0YaLPp+X9t2trUEzEjgBCXqqzEzBjB7T4rbzhAuvU3NRHiAoWXSnrY5AQOTTvC/wpaZV/gF2NDNThjMcpcEdNstleOctXtN4XTTJQewbG+t7I9knR00TPpsoUvUtpbcoFOluGgWTDTFa6GCmDPppx23nZgulOniZUVTI2CbScSpc8sfp1JE7Q6vLECY9lJ+GpVCbDAbIEE9WHXSkJ/6L9yf+ZoHbpqCkEFDcWLfz3B5NFoJ5gne8B8uzd6ptKVtJrP8pMvoOyydyvQlmkgh8tNDsOnp6kaUNG0NHmtfGlCTkbM7oKsKbf87Cv1HgYuZl/HP7go5sOX02LfBfbvWrawZzOaZ3dewEq74w5hVMKEHrmJPstIpcHSxZpq33f2j4Z/uKW9tJat6rgSWK1I+T3tCd++76apxbOYl5TDOaoxCyjQ== lars@marowsky-bree.de", "key_options": null, "keyfile": "/home/lmb/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "lmb", "validate_certs": true} 2021-11-11T20:39:09.063 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ksirivad', 'ovpn': 'ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ksirivad", "ovpn": "ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDMDUcToIhs8tiIozgPudiYyD2upEX+VvsiJRti7zrA/1HPIDARYt2+xyWCASm5x5vdohRM7wnBLUlbBVnvr6ZUQvfShNDsS1Uu7hEfYF38MtW+dVwvGlm14J4m1gbjzqVlZvO6bg6rfGo7Qer0bKUlxtqbisQ/kCg/lVWR04CNxdRtZxEpaQ21y4C5jAlrpFTzxQSuiCLEyY3EAKdZUaZElFEcXlna0ysCP8inlzhF402PkSdl9DkkPpg3IR2dUEGqsblWSdLVk19EFavjtGAAnsvaBWWm2rCLNC5RmhGQaS/Z52Aes1nBuT4cI6qLjbEMqAzLwxiOR7l4IPMpK5EcuRK+FaqQG0wMAjxPc95CmxDzi58ZRJ5ERV8vgps73F1AJXWU9Lb1P77BTsLXqiwko2mvPf6e0LwCxQuxYJWc+xvO7Ji011B+nm5eLCCC/LcDtTPjkimTh7nkvUNi/yTKCXi9KwLISAHHd335KnfrL28rU3GahMXs4NjOxuN8eKM= ksirivad@ksirivad\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDS3HkgHb6z+gi4tN5rBL/N+WLJgN8RtUokt0YnRjpor2epRbmqgmEFRbRFqjLK9LiTuLxmkwol1/G56AJms9ixue33M6jVs73nc259LDxn2qj/NCtxeU7xdjSDuhKyItgM0yuiFjWOKSYTghNtwoR26PX7SQVbycD4NN1ZuwUbdMQA31VrLvQR7d0lYKlis2auH2Xb6WifotoEyszZz1KYRwzJvOHtHku/jAyNmTD/8kAsuCjEgUPSyyehAfdWn59tY2gM7aFyj/dtRoAghekv5gkov9zx+zjN6D2LSN7e+DRti05iJMifA3nfmdj7f8aHzXDtsDL0XNkmmT3g9N2OQ8WL4ESOFASXPPj9JGUROealgnaQ+Y64evDClhbt2L7FLbgj5sgKb+XYhpGnAiC+v3t1Rq3HZrstOYf2qSYguIo+zW9gHwzJDqH5YNCpSF4ZDCfkJ94ngl5ptQmZ/FcrH2hBs2cMoyK9EyDpcI9hBeACTzu19kaB6DrOC+UTvBgYL2sNbCvi+caIqhENJICZ21rI4u4EZqu7PVb+Y3CLgSjTEgclGYBZoxlrT/X2QHrIfZF7lLGeeh0bureedCrBg4b61SL7ZU3Me9x1iWOZMSJgVvg+qPSrzZiHbA46k8i0af5Q+geL1BuuGy76zbYd/KR0KiNZnXjdlHyRz76rpQ== kamoltatsirivadhna@Ksirivad-MacBook-Pro.loc 2021-11-11T20:39:09.065 INFO:teuthology.task.ansible.out:al", "key_options": null, "keyfile": "/home/ksirivad/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ksirivad", "validate_certs": true} ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ksirivad', 'ovpn': 'ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ksirivad", "ovpn": "ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDMDUcToIhs8tiIozgPudiYyD2upEX+VvsiJRti7zrA/1HPIDARYt2+xyWCASm5x5vdohRM7wnBLUlbBVnvr6ZUQvfShNDsS1Uu7hEfYF38MtW+dVwvGlm14J4m1gbjzqVlZvO6bg6rfGo7Qer0bKUlxtqbisQ/kCg/lVWR04CNxdRtZxEpaQ21y4C5jAlrpFTzxQSuiCLEyY3EAKdZUaZElFEcXlna0ysCP8inlzhF402PkSdl9DkkPpg3IR2dUEGqsblWSdLVk19EFavjtGAAnsvaBWWm2rCLNC5RmhGQaS/Z52Aes1nBuT4cI6qLjbEMqAzLwxiOR7l4IPMpK5EcuRK+FaqQG0wMAjxPc95CmxDzi58ZRJ5ERV8vgps73F1AJXWU9Lb1P77BTsLXqiwko2mvPf6e0LwCxQuxYJWc+xvO7Ji011B+nm5eLCCC/LcDtTPjkimTh7nkvUNi/yTKCXi9KwLISAHHd335KnfrL28rU3GahMXs4NjOxuN8eKM= ksirivad@ksirivad\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDS3HkgHb6z+gi4tN5rBL/N+WLJgN8RtUokt0YnRjpor2epRbmqgmEFRbRFqjLK9LiTuLxmkwol1/G56AJms9ixue33M6jVs73nc259LDxn2qj/NCtxeU7xdjSDuhKyItgM0yuiFjWOKSYTghNtwoR26PX7SQVbycD4NN1ZuwUbdMQA31VrLvQR7d0lYKlis2auH2Xb6WifotoEyszZz1KYRwzJvOHtHku/jAyNmTD/8kAsuCjEgUPSyyehAfdWn59tY2gM7aFyj/dtRoAghekv5gkov9zx+zjN6D2LSN7e+DRti05iJMifA3nfmdj7f8aHzXDtsDL0XNkmmT3g9N2OQ8WL4ESOFASXPPj9JGUROealgnaQ+Y64evDClhbt2L7FLbgj5sgKb+XYhpGnAiC+v3t1Rq3HZrstOYf2qSYguIo+zW9gHwzJDqH5YNCpSF4ZDCfkJ94ngl5ptQmZ/FcrH2hBs2cMoyK9EyDpcI9hBeACTzu19kaB6DrOC+UTvBgYL2sNbCvi+caIqhENJICZ21rI4u4EZqu7PVb+Y3CLg 2021-11-11T20:39:09.067 INFO:teuthology.task.ansible.out:SjTEgclGYBZoxlrT/X2QHrIfZF7lLGeeh0bureedCrBg4b61SL7ZU3Me9x1iWOZMSJgVvg+qPSrzZiHbA46k8i0af5Q+geL1BuuGy76zbYd/KR0KiNZnXjdlHyRz76rpQ== kamoltatsirivadhna@Ksirivad-MacBook-Pro.local", "key_options": null, "keyfile": "/home/ksirivad/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ksirivad", "validate_certs": true} 2021-11-11T20:39:09.293 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ofriedma', 'ovpn': 'ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ofriedma", "ovpn": "ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC0POsA+0NyxecLo3Y821G6W3+jJPKFMuLSIl4FSLi25vuyuM5+mFGmCNWv8gg93eQhG9Y3SOygqmkaxzxDtBS/V6VNC2f8hZsVw9B+HbnfggvCNQSgjJ5rhvf6IHfx/aHq4IHF+stqdcoWhwR/68wIhGNqtP7Nhqnzk2aN1kG9bU2iVk4CH8XGIwqwET3aWNuCMcHi2AhrvDBpZu7bzo/6ARUJDqEfTEC/XIkzv8/mEkJ5igcycARVy6dChuCwfZc8rhfwQpekwx1Y9/WBFdNQlYkN4fVe8yIqT5VsQ3AcRv21GJk8L7mwB4hF9dVAi4p1cSyxatR3kDifsFc8VQk9 toor@localhost.localdomain", "key_options": null, "keyfile": "/home/ofriedma/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ofriedma", "validate_certs": true} 2021-11-11T20:39:09.304 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ofriedma', 'ovpn': 'ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ofriedma", "ovpn": "ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC0POsA+0NyxecLo3Y821G6W3+jJPKFMuLSIl4FSLi25vuyuM5+mFGmCNWv8gg93eQhG9Y3SOygqmkaxzxDtBS/V6VNC2f8hZsVw9B+HbnfggvCNQSgjJ5rhvf6IHfx/aHq4IHF+stqdcoWhwR/68wIhGNqtP7Nhqnzk2aN1kG9bU2iVk4CH8XGIwqwET3aWNuCMcHi2AhrvDBpZu7bzo/6ARUJDqEfTEC/XIkzv8/mEkJ5igcycARVy6dChuCwfZc8rhfwQpekwx1Y9/WBFdNQlYkN4fVe8yIqT5VsQ3AcRv21GJk8L7mwB4hF9dVAi4p1cSyxatR3kDifsFc8VQk9 toor@localhost.localdomain", "key_options": null, "keyfile": "/home/ofriedma/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ofriedma", "validate_certs": true} 2021-11-11T20:39:09.521 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'h4iresis', 'ovpn': 'h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "h4iresis", "ovpn": "h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDc/8sWrMq+bqKBPmDPplUgdMoY2SQL+57jRSh27PAoYW9rNVR/UgK7EA9oxFLfoi9wJ9+VKU4opeoohOXLWQXXatLvfXjGE3fRYIonQfwJqKrasqQJTGWzRZ9t1/2Cb0VHkoAvipCJzWmgB46X6q73XW5Td1H3b+AINd6E4Rsm1QjBOhk0DMjCdfwrVUZIfGdsEo+5u/AwioHwH+NDJvXTWT8ZigeZBNHJSY/EQn1+15ut0QB2rvXkJasvvDlVbEsPQtM4C3VX1ronmAnKMvCp1O4EtAXaoKSOx7XEN++UAvVn7czDCWk/UFgpzZOqTcQJwg82XtS4sZP23CpNe0G9 h4iresis@fedora", "key_options": null, "keyfile": "/home/h4iresis/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "h4iresis", "validate_certs": true} 2021-11-11T20:39:09.532 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'h4iresis', 'ovpn': 'h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "h4iresis", "ovpn": "h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDc/8sWrMq+bqKBPmDPplUgdMoY2SQL+57jRSh27PAoYW9rNVR/UgK7EA9oxFLfoi9wJ9+VKU4opeoohOXLWQXXatLvfXjGE3fRYIonQfwJqKrasqQJTGWzRZ9t1/2Cb0VHkoAvipCJzWmgB46X6q73XW5Td1H3b+AINd6E4Rsm1QjBOhk0DMjCdfwrVUZIfGdsEo+5u/AwioHwH+NDJvXTWT8ZigeZBNHJSY/EQn1+15ut0QB2rvXkJasvvDlVbEsPQtM4C3VX1ronmAnKMvCp1O4EtAXaoKSOx7XEN++UAvVn7czDCWk/UFgpzZOqTcQJwg82XtS4sZP23CpNe0G9 h4iresis@fedora", "key_options": null, "keyfile": "/home/h4iresis/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "h4iresis", "validate_certs": true} 2021-11-11T20:39:09.751 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'shyukri', 'ovpn': 'shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "shyukri", "ovpn": "shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqhWVTosH12864VGY5S3/k0yKc59GbV0m2WxD28gAf+uvJ1/y8gekD2wDzBv0DWqEekYLV7L68gpapDXyGosQeZ6n5OO/NZFZmW+Xq1P7P1QHJil6kVpwk927lzX440pwNf3+PRvTbLzlbuVFWJRpiykv+juJyIFjzXLJRF67wB2qgZVm5cRs/GOi4fnvTaou/Ae51mvsO8M3JL5cphApFKD/AO/X+3pHBhLjvaW33cYsp0Z4Rce1nm8wxSmvuJRF+HAfjBlknhDl5ZaJGA96RYA+g9WbAXimFprg4xeiRp7liPExQwtVgackJweoC4MahC1glmjC3eOh9dvrrxLnALOLMOkyCSK/tOF/iq2PJaLViObanPI8mnNkMLbdJyzUBf8hRGdSLcw98qxaf0sSZ+vz4VQS6e9CwX7ikkNNsTeYXjnJ+1yU3mfCW1CXJDr7kBDVG2OTLefCpMY8WBAYDiEaK0RW83cBcFIKfecKYd0qStHRaOsvmyWdFYgBh2vM1UWjro5WXlb+7aJPrUp8ZruapkUXUwhjHA7FwGA4ysE0tsOVVFm0Dw8tvyVVkVNm/9EObG28dWctioZ2+ZEzTWgMJU26Q+wOJ/U4qZyIvpMRUgPZc5nM9J8kcmvNWtb6DV9k/6p8h28kiOxPkxLtPP+3TlBCHjq//mhEnJZ/d8w== shshyukriev@suse.com", "key_options": null, "keyfile": "/home/shyukri/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "shyukri", "validate_certs": true} 2021-11-11T20:39:09.762 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'shyukri', 'ovpn': 'shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "shyukri", "ovpn": "shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqhWVTosH12864VGY5S3/k0yKc59GbV0m2WxD28gAf+uvJ1/y8gekD2wDzBv0DWqEekYLV7L68gpapDXyGosQeZ6n5OO/NZFZmW+Xq1P7P1QHJil6kVpwk927lzX440pwNf3+PRvTbLzlbuVFWJRpiykv+juJyIFjzXLJRF67wB2qgZVm5cRs/GOi4fnvTaou/Ae51mvsO8M3JL5cphApFKD/AO/X+3pHBhLjvaW33cYsp0Z4Rce1nm8wxSmvuJRF+HAfjBlknhDl5ZaJGA96RYA+g9WbAXimFprg4xeiRp7liPExQwtVgackJweoC4MahC1glmjC3eOh9dvrrxLnALOLMOkyCSK/tOF/iq2PJaLViObanPI8mnNkMLbdJyzUBf8hRGdSLcw98qxaf0sSZ+vz4VQS6e9CwX7ikkNNsTeYXjnJ+1yU3mfCW1CXJDr7kBDVG2OTLefCpMY8WBAYDiEaK0RW83cBcFIKfecKYd0qStHRaOsvmyWdFYgBh2vM1UWjro5WXlb+7aJPrUp8ZruapkUXUwhjHA7FwGA4ysE0tsOVVFm0Dw8tvyVVkVNm/9EObG28dWctioZ2+ZEzTWgMJU26Q+wOJ/U4qZyIvpMRUgPZc5nM9J8kcmvNWtb6DV9k/6p8h28kiOxPkxLtPP+3TlBCHjq//mhEnJZ/d8w== shshyukriev@suse.com", "key_options": null, "keyfile": "/home/shyukri/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "shyukri", "validate_certs": true} 2021-11-11T20:39:09.979 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'scarvalhojr', 'ovpn': 'scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "scarvalhojr", "ovpn": "scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCxDJtmmcnrbA4AUsmBBZq/M4xNdIbOT+cQQxhUKeCu8F9mht+RznwyaaJbVIlD+UNv9LMhRyzI5oktSalokNltlrtPQMoP52PXH7KFFGnVnexCHAk4nGvEhuZCniO5T+ENm2wFR47lFEQK7lyJJLp5PFiSM7vM1MK6UkMvRmCTRildjaYLyf3EEvBXifbsGmR0tBIXtml6KITCAaT6l0E94055oEGN2G5NhZIvfLcojt5liEkXEHQNdfkUDXM0Go43JZzgsSWDXC/7RYLrAJwz5mq4qCa9YZL/3Spjso37+1oLq7lOQ0TDJUAQXmnW0So/NkzECx4MolVcylhTzmgj sergio@milestones", "key_options": null, "keyfile": "/home/scarvalhojr/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "scarvalhojr", "validate_certs": true} 2021-11-11T20:39:09.990 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'scarvalhojr', 'ovpn': 'scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "scarvalhojr", "ovpn": "scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCxDJtmmcnrbA4AUsmBBZq/M4xNdIbOT+cQQxhUKeCu8F9mht+RznwyaaJbVIlD+UNv9LMhRyzI5oktSalokNltlrtPQMoP52PXH7KFFGnVnexCHAk4nGvEhuZCniO5T+ENm2wFR47lFEQK7lyJJLp5PFiSM7vM1MK6UkMvRmCTRildjaYLyf3EEvBXifbsGmR0tBIXtml6KITCAaT6l0E94055oEGN2G5NhZIvfLcojt5liEkXEHQNdfkUDXM0Go43JZzgsSWDXC/7RYLrAJwz5mq4qCa9YZL/3Spjso37+1oLq7lOQ0TDJUAQXmnW0So/NkzECx4MolVcylhTzmgj sergio@milestones", "key_options": null, "keyfile": "/home/scarvalhojr/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "scarvalhojr", "validate_certs": true} 2021-11-11T20:39:10.209 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rakesh', 'ovpn': 'rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rakesh", "ovpn": "rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDEOigBGMeElb3uHXX36HxLtPJMG+iG/6D7NDFCVR6AyYsmimBQb6SlSIR6SaFULb6HT/IFVk0Jf/z0HoipAP7Xu+wvP/J6LFoMdguxylh1ov8raqxQJrGLl9n64EI+qWINwEM0MKMUnDQ3tmMKBl78LKQVQTYBx1+9cXsT6DE1yhgmV79XIvOOb+v53agSi/kLJF2i9K04kpi1OejXg95Ub2eLMY69Jm8PnLy1Jq8DhC8pz7nrfzbKtOC7HbytTAO1XTlb90lVhqC/1yTMAUc/rpJwoSEHc2epGeZWdMsJhqhX2EqMwcV53M3qcFSGreGNJ/Ncd4/JHxz7q0IU411t rakesh@localhost.localdomain", "key_options": null, "keyfile": "/home/rakesh/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rakesh", "validate_certs": true} 2021-11-11T20:39:10.220 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rakesh', 'ovpn': 'rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rakesh", "ovpn": "rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDEOigBGMeElb3uHXX36HxLtPJMG+iG/6D7NDFCVR6AyYsmimBQb6SlSIR6SaFULb6HT/IFVk0Jf/z0HoipAP7Xu+wvP/J6LFoMdguxylh1ov8raqxQJrGLl9n64EI+qWINwEM0MKMUnDQ3tmMKBl78LKQVQTYBx1+9cXsT6DE1yhgmV79XIvOOb+v53agSi/kLJF2i9K04kpi1OejXg95Ub2eLMY69Jm8PnLy1Jq8DhC8pz7nrfzbKtOC7HbytTAO1XTlb90lVhqC/1yTMAUc/rpJwoSEHc2epGeZWdMsJhqhX2EqMwcV53M3qcFSGreGNJ/Ncd4/JHxz7q0IU411t rakesh@localhost.localdomain", "key_options": null, "keyfile": "/home/rakesh/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rakesh", "validate_certs": true} 2021-11-11T20:39:10.439 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'ryg-', 'ovpn': 'ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ryg-", "ovpn": "ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvRwrsN62CReT0/kOeFEXGA3THhzJ+hwZVpUzo7VMRZeSo6LgNlVGdrQchMHnlwBvuwaVNJlBAkQr+5J5BB3ikrmbcnLHfPutXBBBB6uDfSwwbqjzUY7elTbz9ul6AkegR9K65O44ouD9+WC76D+Tp0maqnP1w0O3h1GVxDHe34LpEwBvZJ+/BSYmBpwRZqeXFRADORPzdtW08PSiRErGHb8j+j5i+br37JfOOh+A/9dHwQw9alBVL2DijwvrPMa7dV5yX00dlMSTL2LLo9IdI4aFPmie8oic0eanFbQDq210ilOR65IGeshKHEKOq0KFFz0nSK9OdO03OkSEN/0d5 ryg@tapir", "key_options": null, "keyfile": "/home/ryg-/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ryg-", "validate_certs": true} 2021-11-11T20:39:10.451 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'ryg-', 'ovpn': 'ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ryg-", "ovpn": "ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvRwrsN62CReT0/kOeFEXGA3THhzJ+hwZVpUzo7VMRZeSo6LgNlVGdrQchMHnlwBvuwaVNJlBAkQr+5J5BB3ikrmbcnLHfPutXBBBB6uDfSwwbqjzUY7elTbz9ul6AkegR9K65O44ouD9+WC76D+Tp0maqnP1w0O3h1GVxDHe34LpEwBvZJ+/BSYmBpwRZqeXFRADORPzdtW08PSiRErGHb8j+j5i+br37JfOOh+A/9dHwQw9alBVL2DijwvrPMa7dV5yX00dlMSTL2LLo9IdI4aFPmie8oic0eanFbQDq210ilOR65IGeshKHEKOq0KFFz0nSK9OdO03OkSEN/0d5 ryg@tapir", "key_options": null, "keyfile": "/home/ryg-/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ryg-", "validate_certs": true} 2021-11-11T20:39:10.669 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "prsrivas", "ovpn": "prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC0zCoXv2Kkxjqv5FOlJZ2PRA9tStdXAVGZJbSLOhAWeqHQL6O2l8MO5GFzHz4FA/g7lbNzLKaCbG1wYGuL9FOt0o36/eHykgENu00SlFunIQn+T/WNaGxrZqnfbdS9TdImFAnVY4vuzrBhFTb6qcqTLhw1WVz+l/iOv5CoY+7NnILWcj8dHNcPzU1ihlFAbvQgQcMtt1130zqc6L8o+SDm2YQF2nuOWxj0LbD6YpRn3AqWsTN5QbqzdeaJiUt6kdywisPdoiSWLuivRKhNLNhnWO+B4DnchwfApy8ovJShHZAsrSoc+a+RQDmOiLXmc5wAvNnREHbqod8/0bQ9vSXaVQCt1gnnK8Bf7GQElg3Z3aep4D4w6rBB+LBsHFbTfrzWAHkXxJeGiQTlJGz0IYHi1weWrzY6x5rEA0gYDu0FVLOLrsYDtuTQpjcTwpmaHCsxz6wU3wCNmTsV/AZhniFHJhHXacCmDp1JfsI2kP9FjcJJE0SygnZ/SzKe0DHamnyT1OgdQaG/KX2eRv/vXUPg2ukKiA1fyFnFeEsWK6rUO0YgCswqjrM4vU0leZ1b8vK9aM4IDJM3TJqdMLfOa0rPntekNOqXUn213CP+Yiz9l1TH4zX3P/Kv6Vv1gTeGcX0lMaYhowng+97j3LLqGcSUPMklrKen/xrsACFxN9fxaQ== prsrivas@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCks+XmOepYEdSBP6U0T+r+fZowSS7UHY5W8AUtAkvXiKlTG518luG2wtSALIsB5+jyjC44ITwV9X/vvsDaXQnp1hFanwWeuU1jDdy+MvywvxI9q4naWlDpxrfq4i/AioSvdoIo02uUFYPKTCHrm7wBjTwpWbCD0gbC9/OGAHM88Md3F0qbgFEr3BERvNGe0coCjWRMS8+GTZXS22WW1/NxNzpdCUjO/9j4QydpW7UF1kqVhnPVZozmh4wl0rk8tgb9Zx3vCUBG1Q5eO/q+HsW54eXLa9FcuQcoJnNdV2hMXRHHZHTgaPKZ/ildukzZHo0i/Dq1mi5N3s5A7aDYxOWWlRXazwL8zu/UQVrxVRni8vDTFk5682lskF1WifzADaqicUWzasdK0PC4G7JmHPi+XHskeOt5i4ywEnFRXnYtMOIu0CNO82Rg4CdUAD58YXCGoQ4r9TG4LlwKg75ouNHB/ToJidSg9NBwclP+MICcTpWItxVXpBj/YkeLcntHtVTFNHORxIwIwWng94wVEDCfyXelyAR6VfwcPlWWrbzhbEO9DH40GuluQi38ySWi/HpnrVriJMLzEQA9OPrLsME1LDgPGkF2GloPL8nnnLWB9L+YEBNNAVoSo0PHBsjIibdZ/SZm7zUgYGp4fuG+m1+SDOjcfD6GSXOTQwQou+knEQ== prsrivas@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDISEXvGI2oJLTRR15ePZLkmdYjaQJxt9V/ 2021-11-11T20:39:10.671 INFO:teuthology.task.ansible.out:yBwKdGqYvnPzmLOpdmVpdQJo7ZOWwdp5tJsCy4q5uX5vIUv6Gyi3uVSGmZnrKUbsqoDNwAX5vx/gFBINIs/ZXXyD4fpUT8aoZeq1XmXdIaWG0nImsrtRnRxKP9NGZEcW8z0WPwgF0Ck+RH0qZUix6Bfbgy7vMXnivuDv+NdjEwyszcrPWvf6Krr0am9Q0+9nj/mjvk6CZLl+CH3HNdkhZuNs+qMG4XW8KhLvQB2p790XVlTHuVU4jWrFmjymggZxkhoCY/ukHH6wi2Pbx6yjjKOj8iIw1xKabJzJ7S8UKT23A41d918DUCUuEG66pN6Z4dnv+fZJxj5iEo1GdycSWIc4SpLTK3z4zDZypVKIzjI4o8VwQaN48HkVjUdTICTvGQfytFi3jX1I5nxIZiiPcxMQLR5hQggP8pn2XtvXpomn0aBc8tVuuZJLVq4zCVnhlVB6IACmb71evjFGOsoSuEsaTufUr48= prsrivas@t480.lab.blr.redhat.com", "key_options": null, "keyfile": "/home/prsrivas/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "prsrivas", "validate_certs": true} 2021-11-11T20:39:10.680 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "prsrivas", "ovpn": "prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC0zCoXv2Kkxjqv5FOlJZ2PRA9tStdXAVGZJbSLOhAWeqHQL6O2l8MO5GFzHz4FA/g7lbNzLKaCbG1wYGuL9FOt0o36/eHykgENu00SlFunIQn+T/WNaGxrZqnfbdS9TdImFAnVY4vuzrBhFTb6qcqTLhw1WVz+l/iOv5CoY+7NnILWcj8dHNcPzU1ihlFAbvQgQcMtt1130zqc6L8o+SDm2YQF2nuOWxj0LbD6YpRn3AqWsTN5QbqzdeaJiUt6kdywisPdoiSWLuivRKhNLNhnWO+B4DnchwfApy8ovJShHZAsrSoc+a+RQDmOiLXmc5wAvNnREHbqod8/0bQ9vSXaVQCt1gnnK8Bf7GQElg3Z3aep4D4w6rBB+LBsHFbTfrzWAHkXxJeGiQTlJGz0IYHi1weWrzY6x5rEA0gYDu0FVLOLrsYDtuTQpjcTwpmaHCsxz6wU3wCNmTsV/AZhniFHJhHXacCmDp1JfsI2kP9FjcJJE0SygnZ/SzKe0DHamnyT1OgdQaG/KX2eRv/vXUPg2ukKiA1fyFnFeEsWK6rUO0YgCswqjrM4vU0leZ1b8vK9aM4IDJM3TJqdMLfOa0rPntekNOqXUn213CP+Yiz9l1TH4zX3P/Kv6Vv1gTeGcX0lMaYhowng+97j3LLqGcSUPMklrKen/xrsACFxN9fxaQ== prsrivas@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCks+XmOepYEdSBP6U0T+r+fZowSS7UHY5W8AUtAkvXiKlTG518luG2wtSALIsB5+jyjC44ITwV9X/vvsDaXQnp1hFanwWeuU1jDdy+MvywvxI9q4naWlDpxrfq4i/AioSvdoIo02uUFYPKTCHrm7wBjTwpWbCD0gbC9/OGAHM88Md3F0qbgFEr3BERvNGe0coCjWRMS8+GTZXS22WW1/NxNzpdCUjO/9j4QydpW7UF1kqVhnPVZozmh4wl0rk8tgb9Zx3vCUBG1Q5eO/q+HsW54eXLa9FcuQcoJnNdV2hMXRHHZHTgaPKZ/ildukzZHo0i/Dq1mi5N3s5A7aDYxOWWlRXazwL8zu/UQVrxVRni8vDTFk5682lskF1WifzADaqicUWzasdK0PC4G7JmHPi+XHskeOt5i4ywEnFRXnYtMOIu0CNO82Rg4CdUAD58YXCGoQ4r9TG4LlwKg75ouNHB/ToJidSg9NBwclP+MICcTpWItxVXpBj/YkeLcntHtVTFNHORxIwIwWng94wVEDCfyXelyAR6VfwcPlWWrbzhbEO9DH40GuluQi38ySWi/HpnrVriJMLzEQA9OPrLsME1LDgPGkF2GloPL8nnnLWB9L+YEBNNAVoSo0PHBsjIibdZ/SZm7zUgYGp4fuG+m1+SDOjcfD6GSXOTQwQou+knEQ== prsrivas@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDISEXvGI2oJLTRR15ePZLkmdYjaQJxt9V/ 2021-11-11T20:39:10.682 INFO:teuthology.task.ansible.out:yBwKdGqYvnPzmLOpdmVpdQJo7ZOWwdp5tJsCy4q5uX5vIUv6Gyi3uVSGmZnrKUbsqoDNwAX5vx/gFBINIs/ZXXyD4fpUT8aoZeq1XmXdIaWG0nImsrtRnRxKP9NGZEcW8z0WPwgF0Ck+RH0qZUix6Bfbgy7vMXnivuDv+NdjEwyszcrPWvf6Krr0am9Q0+9nj/mjvk6CZLl+CH3HNdkhZuNs+qMG4XW8KhLvQB2p790XVlTHuVU4jWrFmjymggZxkhoCY/ukHH6wi2Pbx6yjjKOj8iIw1xKabJzJ7S8UKT23A41d918DUCUuEG66pN6Z4dnv+fZJxj5iEo1GdycSWIc4SpLTK3z4zDZypVKIzjI4o8VwQaN48HkVjUdTICTvGQfytFi3jX1I5nxIZiiPcxMQLR5hQggP8pn2XtvXpomn0aBc8tVuuZJLVq4zCVnhlVB6IACmb71evjFGOsoSuEsaTufUr48= prsrivas@t480.lab.blr.redhat.com", "key_options": null, "keyfile": "/home/prsrivas/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "prsrivas", "validate_certs": true} 2021-11-11T20:39:10.908 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'zdover', 'ovpn': 'zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "zdover", "ovpn": "zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDOJg56HsyNa5vHHb52G6O5vtv9I2tdlNMjFlG3SVCCu/bVZwVeGzp1ekuNGU10owmnIvhlSldMjLn2wwTwiDjbvYJVVCPvzK63vCUcMevRsvr9EyU16JdAgrmrGMyuYYEb+wh8XWI9Dc72o84fMaBYFSSer1S9nUBL+KVLVhIe7culgADimrDgDVh2t7u2JHBI0m0nKFGJZNcIyYf+GKOM+n7SaQXIYIytvSteU5BvUIKPsDd4z5lVlWiKO3kG6jF2RnKFV6KWBWI9m8YGItdGu25mXELXN9MOlApZF3AoGTmCE2Web7uDVGvO6rz6b+dc7RXTOlbWIhtMoRqQf6mz zac.dover@gmail.com", "key_options": null, "keyfile": "/home/zdover/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "zdover", "validate_certs": true} 2021-11-11T20:39:10.910 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'zdover', 'ovpn': 'zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "zdover", "ovpn": "zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDOJg56HsyNa5vHHb52G6O5vtv9I2tdlNMjFlG3SVCCu/bVZwVeGzp1ekuNGU10owmnIvhlSldMjLn2wwTwiDjbvYJVVCPvzK63vCUcMevRsvr9EyU16JdAgrmrGMyuYYEb+wh8XWI9Dc72o84fMaBYFSSer1S9nUBL+KVLVhIe7culgADimrDgDVh2t7u2JHBI0m0nKFGJZNcIyYf+GKOM+n7SaQXIYIytvSteU5BvUIKPsDd4z5lVlWiKO3kG6jF2RnKFV6KWBWI9m8YGItdGu25mXELXN9MOlApZF3AoGTmCE2Web7uDVGvO6rz6b+dc7RXTOlbWIhtMoRqQf6mz zac.dover@gmail.com", "key_options": null, "keyfile": "/home/zdover/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "zdover", "validate_certs": true} 2021-11-11T20:39:11.138 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'rraja', 'ovpn': 'rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rraja", "ovpn": "rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC4zwWf+kMEDgaSOrX7AlD+UCOgf4/fRiorQVJqC8GfMNynfiFVZaThngwIrwtQROfXHbzspi+AcRvDLewY25mXyuxVgOfSbuSIEqiUNvzcVq9IItI1J58xeDBpIktAsoHHS6JENV56fAT6krzRAQLnSa1k2b0pyKuPBzFSeNgiaQMhjrCLAaCi9yLsdKpnWgXhyfrdqvN2r/Ctts/zlTPqtRYdvdIL/p/l+vg7Q8S3OhuAGtimaEq3RE70xadaPbGhsLISMTrbfYXbxw84KKZpNYwO4zmQ2ztcvC9D7A/8ov3+A5Jq42G+pq7ORuix2NoTWtADMiwvNe1HyzS/7cX/BuXLFssRY1bzRvp+x2ExZ7b+bFx+TE/RduRh0eyDNLg7SyL83aeTr2Usa+mPoCAMKed9XtYbRnUu7ox4wmfeAgx6jM9uG5uQfauJ1XpDddrVBG8KYBvSd0BM2cyRdcAXqwxg9AVPZcdE7sbBk44CGVhw+ziZ3e3cLBTO4y44KRQre+1Bhe/Ze8OTQWLFugl/XagFlJE/L6fGUIPuFojmQvlw5dBZwtUURnxsu8yHm3iIsj4y8nR1zHMj+zuMk6rF3ZW0epPAECAdq4gG8X2wv3HTq0vLBkKpi1KMAmntEtcVwF0fzMt5XYqRd4jceFgonAAipJNDknHgi6yaUzBYmQ== rraja@redhat.com", "key_options": null, "keyfile": "/home/rraja/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rraja", "validate_certs": true} 2021-11-11T20:39:11.140 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'rraja', 'ovpn': 'rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "rraja", "ovpn": "rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC4zwWf+kMEDgaSOrX7AlD+UCOgf4/fRiorQVJqC8GfMNynfiFVZaThngwIrwtQROfXHbzspi+AcRvDLewY25mXyuxVgOfSbuSIEqiUNvzcVq9IItI1J58xeDBpIktAsoHHS6JENV56fAT6krzRAQLnSa1k2b0pyKuPBzFSeNgiaQMhjrCLAaCi9yLsdKpnWgXhyfrdqvN2r/Ctts/zlTPqtRYdvdIL/p/l+vg7Q8S3OhuAGtimaEq3RE70xadaPbGhsLISMTrbfYXbxw84KKZpNYwO4zmQ2ztcvC9D7A/8ov3+A5Jq42G+pq7ORuix2NoTWtADMiwvNe1HyzS/7cX/BuXLFssRY1bzRvp+x2ExZ7b+bFx+TE/RduRh0eyDNLg7SyL83aeTr2Usa+mPoCAMKed9XtYbRnUu7ox4wmfeAgx6jM9uG5uQfauJ1XpDddrVBG8KYBvSd0BM2cyRdcAXqwxg9AVPZcdE7sbBk44CGVhw+ziZ3e3cLBTO4y44KRQre+1Bhe/Ze8OTQWLFugl/XagFlJE/L6fGUIPuFojmQvlw5dBZwtUURnxsu8yHm3iIsj4y8nR1zHMj+zuMk6rF3ZW0epPAECAdq4gG8X2wv3HTq0vLBkKpi1KMAmntEtcVwF0fzMt5XYqRd4jceFgonAAipJNDknHgi6yaUzBYmQ== rraja@redhat.com", "key_options": null, "keyfile": "/home/rraja/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "rraja", "validate_certs": true} 2021-11-11T20:39:11.367 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'toabctl', 'ovpn': 'toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "toabctl", "ovpn": "toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2mEyxfkW1u84u5BocLCAXESj2qObWkxUd20cmPolhJYlIWLOafa69Dw5A9CjFBp0U6Jx5d0JWcwmAc5ZeffQ7t7vzmtHCZnpKkyZhN9bu6vuCh4N+I643SI/K7tdbB2E8ZE6uI7ORMAan5+t0bNSVolUVq6X1w7dIA5El54K10WNdiwgRKW6e3anNbJ3rBLGEk7gawZqDDOqTvVQmOBGWek3BjSReDvg3rkmTQLxJkiKdxnx4/s7gzH1xTpsS/OludIuSxF0BK064ZcT1t09qcjnT4Os1inV2GfYswt+UhpgqrHqXjEs0Z3y4Oy77QBKL1rDf93CP4a4+eJjqXe+v tbechtold@suse.com", "key_options": null, "keyfile": "/home/toabctl/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "toabctl", "validate_certs": true} 2021-11-11T20:39:11.369 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'toabctl', 'ovpn': 'toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "toabctl", "ovpn": "toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2mEyxfkW1u84u5BocLCAXESj2qObWkxUd20cmPolhJYlIWLOafa69Dw5A9CjFBp0U6Jx5d0JWcwmAc5ZeffQ7t7vzmtHCZnpKkyZhN9bu6vuCh4N+I643SI/K7tdbB2E8ZE6uI7ORMAan5+t0bNSVolUVq6X1w7dIA5El54K10WNdiwgRKW6e3anNbJ3rBLGEk7gawZqDDOqTvVQmOBGWek3BjSReDvg3rkmTQLxJkiKdxnx4/s7gzH1xTpsS/OludIuSxF0BK064ZcT1t09qcjnT4Os1inV2GfYswt+UhpgqrHqXjEs0Z3y4Oy77QBKL1rDf93CP4a4+eJjqXe+v tbechtold@suse.com", "key_options": null, "keyfile": "/home/toabctl/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "toabctl", "validate_certs": true} 2021-11-11T20:39:11.596 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mgfritch', 'ovpn': 'mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mgfritch", "ovpn": "mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCitWIN7jm5KEmB6jqcA+ufKRLodxkJ1GIdaaueCdeXdMHgWNH9FUDCUJk5dcA11MxQO15DRH/cdhb3QHPGiMv4XrBeK6HHYwGttzkcUEz1EDUhpowOrnFETbPa7K39ObmRcgdvoGXOSrNZ4mvptCTz9Xb0MF+Gcg5iJ4g+qZzIVa2/ueecYY8ZorbaH4m/NWUub6Rf01Q04HfxTvKrUFswrbCoE/Aigz/Oh88RiS854XoTvV7rE1tKbyKkaungwMAaQSNnZkq4hi8nbp6pzTxJLV7N/ZYMhVpdxSbWcghFg6cy7aeoCzZMN97WcDhpdfbhRT0dbqCkMmHV6t0usDDnOss2v68UCzkxuA1Hj4uy8rRHjXBtYqI6xsrwCjvmgP05B14TpmFdG+3GdRHx2+QlhnGj7nuPf1KoWW/f3tD85DXoEmVl6RJ4KTrMeEVlOn3kdMLVH0y2hIqPTYMP5cMVNi0EdfeoBdd9kxMKMvoSVBAgmLzGVv17o1lNvVS6V1kyt7gME2PfNR2Np+U1cJVJfUJRTm28mHXBVoDXG9s0zv8+LpQ/gVPS+vsPAfxWWYGW8ZmEV3ZorDOZycSPybjHkPFzDAdUhPG0Db0o+F9ug+22F7pvifsdDZ42zqGGLbyzfIQE3AtyI7LR7Sc5tpN9rKTeSBYJ0yNcDKEj5FUHDw== mfritch@suse.com", "key_options": null, "keyfile": "/home/mgfritch/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mgfritch", "validate_certs": true} 2021-11-11T20:39:11.598 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mgfritch', 'ovpn': 'mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mgfritch", "ovpn": "mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCitWIN7jm5KEmB6jqcA+ufKRLodxkJ1GIdaaueCdeXdMHgWNH9FUDCUJk5dcA11MxQO15DRH/cdhb3QHPGiMv4XrBeK6HHYwGttzkcUEz1EDUhpowOrnFETbPa7K39ObmRcgdvoGXOSrNZ4mvptCTz9Xb0MF+Gcg5iJ4g+qZzIVa2/ueecYY8ZorbaH4m/NWUub6Rf01Q04HfxTvKrUFswrbCoE/Aigz/Oh88RiS854XoTvV7rE1tKbyKkaungwMAaQSNnZkq4hi8nbp6pzTxJLV7N/ZYMhVpdxSbWcghFg6cy7aeoCzZMN97WcDhpdfbhRT0dbqCkMmHV6t0usDDnOss2v68UCzkxuA1Hj4uy8rRHjXBtYqI6xsrwCjvmgP05B14TpmFdG+3GdRHx2+QlhnGj7nuPf1KoWW/f3tD85DXoEmVl6RJ4KTrMeEVlOn3kdMLVH0y2hIqPTYMP5cMVNi0EdfeoBdd9kxMKMvoSVBAgmLzGVv17o1lNvVS6V1kyt7gME2PfNR2Np+U1cJVJfUJRTm28mHXBVoDXG9s0zv8+LpQ/gVPS+vsPAfxWWYGW8ZmEV3ZorDOZycSPybjHkPFzDAdUhPG0Db0o+F9ug+22F7pvifsdDZ42zqGGLbyzfIQE3AtyI7LR7Sc5tpN9rKTeSBYJ0yNcDKEj5FUHDw== mfritch@suse.com", "key_options": null, "keyfile": "/home/mgfritch/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mgfritch", "validate_certs": true} 2021-11-11T20:39:11.836 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'xiubli', 'ovpn': 'root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "xiubli", "ovpn": "root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5bCXQ/B16AKl2oevC93R0U+3OrTyd3XEypouJy4LjIzXLX/Go1C1ViERwDAeHmqiUTqJUL+KZ0kMo8U7zlhNBk9PucbMZCqdqx5hkiCAo4y65lEXmuPGimRU2mK9TpcDTLWoyrD/EEhwzT6gmw3ytVTxi7qAb0tQUCQB+pW4+Zr5u6HZzgu9kLGxp7zsGtMA84Zqm5ql72mGuS0bUN1ajm8LWBetFuzOOW6YGfnKt29cLV2h1itVewA9DKM5G70B91eK4LjoNddCkqVA8L+v6LMQNU2pejUzPap98hmFUpQMNxlWQ0+B7UjfbkHWe+mIsSkdr3FfCfqWvsL0RiHcZ root@lxbceph0", "key_options": null, "keyfile": "/home/xiubli/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "xiubli", "validate_certs": true} 2021-11-11T20:39:11.838 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'xiubli', 'ovpn': 'root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "xiubli", "ovpn": "root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5bCXQ/B16AKl2oevC93R0U+3OrTyd3XEypouJy4LjIzXLX/Go1C1ViERwDAeHmqiUTqJUL+KZ0kMo8U7zlhNBk9PucbMZCqdqx5hkiCAo4y65lEXmuPGimRU2mK9TpcDTLWoyrD/EEhwzT6gmw3ytVTxi7qAb0tQUCQB+pW4+Zr5u6HZzgu9kLGxp7zsGtMA84Zqm5ql72mGuS0bUN1ajm8LWBetFuzOOW6YGfnKt29cLV2h1itVewA9DKM5G70B91eK4LjoNddCkqVA8L+v6LMQNU2pejUzPap98hmFUpQMNxlWQ0+B7UjfbkHWe+mIsSkdr3FfCfqWvsL0RiHcZ root@lxbceph0", "key_options": null, "keyfile": "/home/xiubli/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "xiubli", "validate_certs": true} 2021-11-11T20:39:12.064 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jafaj', 'ovpn': 'jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jafaj", "ovpn": "jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDT+6B5Fr99p8xe9Zo5x8ULcdeA6ne727iVJp/xaguGwGHj6AhuK+R/CjznnHk0yA+6kiblNzjvbnsuzO8T9c8Ayv5KU07MxP1y1pymdYBpEh3NTXmhB6QOTCNTQbT14PL6MVgzK17SqarFCYXoEGuyK9jaBdN9QM76vhEaXc/2Osg+d8boJTkOk7lOPHbzeOCbQU3d+jX0VFOgAQS5OFz3Ug/X7nB0D0LN8bt5FlCZiGO95Iwls/SyPDbVXc9ot8oh4ME7kd7sElokBdYdXPauibRxWKjSDn+hV6kZfSzqwnMFaZ1TC+Av6zNsBuZucNSp3/Ws3T5i9i+21eyS4+qVE8IiX2aVKYuC9afb2lPLbdG2Iy1CqiaeGPkZfiGT9P8iQPOkqae45859UzFAd/XzKjBcmus3lSArKG1EyZl03HQ5P2J60pErCXq8W9mPjxAze2ReLRXJVEgTXnmdhCptcg6PZ4d/k4BEddbdnoX6/BEgFxLEJ64HpTeTVcDXHpipCpxFSRvgyPH/wQGKAvt8AfKBhmeZs7LuyGOhJiOon4DDaShHpE9wTnJ5qlnyg44+u6NCwelGPMAkHKEO4OgcjSsyXm+HYUpxG2s4poloIJzvU6oa3dvJsMZFhcb8CsW75sBYE+YngS/1fg1WU88ZOdaa1yF4F87pVZycpZqY1w== jan@jfsuselaptop", "key_options": null, "keyfile": "/home/jafaj/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jafaj", "validate_certs": true} 2021-11-11T20:39:12.066 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jafaj', 'ovpn': 'jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jafaj", "ovpn": "jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDT+6B5Fr99p8xe9Zo5x8ULcdeA6ne727iVJp/xaguGwGHj6AhuK+R/CjznnHk0yA+6kiblNzjvbnsuzO8T9c8Ayv5KU07MxP1y1pymdYBpEh3NTXmhB6QOTCNTQbT14PL6MVgzK17SqarFCYXoEGuyK9jaBdN9QM76vhEaXc/2Osg+d8boJTkOk7lOPHbzeOCbQU3d+jX0VFOgAQS5OFz3Ug/X7nB0D0LN8bt5FlCZiGO95Iwls/SyPDbVXc9ot8oh4ME7kd7sElokBdYdXPauibRxWKjSDn+hV6kZfSzqwnMFaZ1TC+Av6zNsBuZucNSp3/Ws3T5i9i+21eyS4+qVE8IiX2aVKYuC9afb2lPLbdG2Iy1CqiaeGPkZfiGT9P8iQPOkqae45859UzFAd/XzKjBcmus3lSArKG1EyZl03HQ5P2J60pErCXq8W9mPjxAze2ReLRXJVEgTXnmdhCptcg6PZ4d/k4BEddbdnoX6/BEgFxLEJ64HpTeTVcDXHpipCpxFSRvgyPH/wQGKAvt8AfKBhmeZs7LuyGOhJiOon4DDaShHpE9wTnJ5qlnyg44+u6NCwelGPMAkHKEO4OgcjSsyXm+HYUpxG2s4poloIJzvU6oa3dvJsMZFhcb8CsW75sBYE+YngS/1fg1WU88ZOdaa1yF4F87pVZycpZqY1w== jan@jfsuselaptop", "key_options": null, "keyfile": "/home/jafaj/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jafaj", "validate_certs": true} 2021-11-11T20:39:12.293 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jschmid', 'ovpn': 'jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jschmid", "ovpn": "jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDo8wE/b2/EDwV018ENfcENVfXSF7vnXAwBSzNVScgxdqr2DVKTWDzhoI0m5CXTqI2HLnJVFDTgN3t5/fFcmwmSAP8ic8wU1JznSPZXN7qEQrcEsU/cwHeuzyutibg1ItTWWtoXYV8SFnVom8NF3UwXF8I+Vj6VF1EStWCcClgPjLbiOm8vIgaouXtbUIFv25GpadXyUtwiHEq1NKUPLgka8Usm8ibyS9hY9KmNWlD9AfmnwAImc8Urn2dErcsfAltxoGUHCI7ZBHGpzg+6dpynNR+giJMrjwWzREaCHmiJX3g6jmHuuY54eSOs6cxqi3E4jNHVWXPe/tEF3l0klEvl jschmid@suse.com", "key_options": null, "keyfile": "/home/jschmid/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jschmid", "validate_certs": true} 2021-11-11T20:39:12.295 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jschmid', 'ovpn': 'jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jschmid", "ovpn": "jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDo8wE/b2/EDwV018ENfcENVfXSF7vnXAwBSzNVScgxdqr2DVKTWDzhoI0m5CXTqI2HLnJVFDTgN3t5/fFcmwmSAP8ic8wU1JznSPZXN7qEQrcEsU/cwHeuzyutibg1ItTWWtoXYV8SFnVom8NF3UwXF8I+Vj6VF1EStWCcClgPjLbiOm8vIgaouXtbUIFv25GpadXyUtwiHEq1NKUPLgka8Usm8ibyS9hY9KmNWlD9AfmnwAImc8Urn2dErcsfAltxoGUHCI7ZBHGpzg+6dpynNR+giJMrjwWzREaCHmiJX3g6jmHuuY54eSOs6cxqi3E4jNHVWXPe/tEF3l0klEvl jschmid@suse.com", "key_options": null, "keyfile": "/home/jschmid/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jschmid", "validate_certs": true} 2021-11-11T20:39:12.533 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kachau', 'ovpn': 'kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kachau", "ovpn": "kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnQfkkxYWLhMdoQ4RsZxWTXBpbzsssgeRfvV5q+n0mW60dxQ+9rjDXUTsRkmh45B5EhNWlKwyIUcQ5HpgJk6XQvYU6MPboWjriCTngmoUbwNIjOjyazkBbxfbwUiRDb+x1EyVm4Kt0rgj3nhqqxf3krqO2SlKgsVaY9b7+NTfshXb9T73/RRLtvwUOFkjp2B+w0SQs7Ka1GenmHSY/6xP25Mb8q9QW/2nuf1mV9rTd5l2ZHREzgG7gQFhjgUq3w6zCCfcCGwrPb4omqd9xiYLDw8l1vfQ7UPbuOqK1rFM67r7JQDG3yyJjCdWgRUI/ysbbXj0fyT+QOPZGK618V8yZPVlq5loQzYI39zAWZU9L9X9bkQCsJfjU5Yo9f/zOaecnWfUxC84NKpbG7k6XACdUIB9JGGtL6RPtgBU+k6Ts7EemIUCyMYm/GX+PYU3GnmmdqW9yBeyZoldwDCWLq05vjLXj+R3J2UrMl5x+fhP+KEFd1ZgNc7PV+CF35/ux8UE= kachau@localhost.localdomain", "key_options": null, "keyfile": "/home/kachau/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kachau", "validate_certs": true} 2021-11-11T20:39:12.534 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kachau', 'ovpn': 'kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kachau", "ovpn": "kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnQfkkxYWLhMdoQ4RsZxWTXBpbzsssgeRfvV5q+n0mW60dxQ+9rjDXUTsRkmh45B5EhNWlKwyIUcQ5HpgJk6XQvYU6MPboWjriCTngmoUbwNIjOjyazkBbxfbwUiRDb+x1EyVm4Kt0rgj3nhqqxf3krqO2SlKgsVaY9b7+NTfshXb9T73/RRLtvwUOFkjp2B+w0SQs7Ka1GenmHSY/6xP25Mb8q9QW/2nuf1mV9rTd5l2ZHREzgG7gQFhjgUq3w6zCCfcCGwrPb4omqd9xiYLDw8l1vfQ7UPbuOqK1rFM67r7JQDG3yyJjCdWgRUI/ysbbXj0fyT+QOPZGK618V8yZPVlq5loQzYI39zAWZU9L9X9bkQCsJfjU5Yo9f/zOaecnWfUxC84NKpbG7k6XACdUIB9JGGtL6RPtgBU+k6Ts7EemIUCyMYm/GX+PYU3GnmmdqW9yBeyZoldwDCWLq05vjLXj+R3J2UrMl5x+fhP+KEFd1ZgNc7PV+CF35/ux8UE= kachau@localhost.localdomain", "key_options": null, "keyfile": "/home/kachau/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kachau", "validate_certs": true} 2021-11-11T20:39:12.763 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'khiremat', 'ovpn': 'khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "khiremat", "ovpn": "khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQujcXnq9qOCmTWs44GwKOw48BxZGBmo9AeFk3KPRx33Cz5hS7UHPYvJy03KdoMySgxJtuxsOSvjJSPxpXesTtArO5giW8RLyuVsu9q5j/GhPKyABttuGQyexiRokrHQFLVifzgqwsUARwHarWH16Oa1n6fZPFNqSH56c872zS4Pwqgkzx99NKRWh/B+fIk8VmDzzP1qvQAnUXDeTeTOzapAL+8fSNTp3QnhmZbvCHYwUxfSqJyzq1wBL+517WrvzvX7yCY9B0wzm72sIO4daV6UDyAL1B72QpB04vEZZ/skwS/0jmokfBn43HHCMPy/Mxywfi54alFJzIu4pti30V kravishankar@localhost.localdomain", "key_options": null, "keyfile": "/home/khiremat/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "khiremat", "validate_certs": true} 2021-11-11T20:39:12.765 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'khiremat', 'ovpn': 'khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "khiremat", "ovpn": "khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQujcXnq9qOCmTWs44GwKOw48BxZGBmo9AeFk3KPRx33Cz5hS7UHPYvJy03KdoMySgxJtuxsOSvjJSPxpXesTtArO5giW8RLyuVsu9q5j/GhPKyABttuGQyexiRokrHQFLVifzgqwsUARwHarWH16Oa1n6fZPFNqSH56c872zS4Pwqgkzx99NKRWh/B+fIk8VmDzzP1qvQAnUXDeTeTOzapAL+8fSNTp3QnhmZbvCHYwUxfSqJyzq1wBL+517WrvzvX7yCY9B0wzm72sIO4daV6UDyAL1B72QpB04vEZZ/skwS/0jmokfBn43HHCMPy/Mxywfi54alFJzIu4pti30V kravishankar@localhost.localdomain", "key_options": null, "keyfile": "/home/khiremat/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "khiremat", "validate_certs": true} 2021-11-11T20:39:12.993 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kapandya', 'ovpn': 'kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kapandya", "ovpn": "kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCoqsDnRqAeNPi5xiIfFl2i2zqcwLTvcAso3R9+QAfSChw4+wcQzmMV8uQz1jt8UReefg2QL6bwvcQPj5Zw7ADqUjfxm1K0ezgH6RrtHgRZ1SVseTvJUvzmkDZ2gr2ToZAJYsaV7gBLJLZ7TOiQ6jzrtitiKqJZdpZfzadjShecY1TdMo8bSS7XfbRcwri40u82TRM825B4L//QuhMI7akPqa3iJbGEAQPhkGLI3kz19ryVG1UwvAkkt7TQ68zz+piXSmB2CMRti51shT4NuAOq6Qe9y2Obicf88ItkYxvySMQ5jWi2bo7Zhwuq9jUH8lK6I6PHrYc8LOsR9gE/hqkXxTQrkOXEDtHN1mr/KKLrXO0GeBdgh8Gj4gcwgd1RUCE+T9q09WO9j7c0piWK9Fs1+q1FjH06/AfmP4qob1TlJMHhtPyOp+lk2XvABP+sqW4vyrgmXu5vJU+Hgq3X9KE7bKh4WkCem24Np4uA+irhpvZmIoHyINsYW46lGfwJvM8= kapandya@localhost.localdomain", "key_options": null, "keyfile": "/home/kapandya/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kapandya", "validate_certs": true} 2021-11-11T20:39:12.993 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kapandya', 'ovpn': 'kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kapandya", "ovpn": "kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCoqsDnRqAeNPi5xiIfFl2i2zqcwLTvcAso3R9+QAfSChw4+wcQzmMV8uQz1jt8UReefg2QL6bwvcQPj5Zw7ADqUjfxm1K0ezgH6RrtHgRZ1SVseTvJUvzmkDZ2gr2ToZAJYsaV7gBLJLZ7TOiQ6jzrtitiKqJZdpZfzadjShecY1TdMo8bSS7XfbRcwri40u82TRM825B4L//QuhMI7akPqa3iJbGEAQPhkGLI3kz19ryVG1UwvAkkt7TQ68zz+piXSmB2CMRti51shT4NuAOq6Qe9y2Obicf88ItkYxvySMQ5jWi2bo7Zhwuq9jUH8lK6I6PHrYc8LOsR9gE/hqkXxTQrkOXEDtHN1mr/KKLrXO0GeBdgh8Gj4gcwgd1RUCE+T9q09WO9j7c0piWK9Fs1+q1FjH06/AfmP4qob1TlJMHhtPyOp+lk2XvABP+sqW4vyrgmXu5vJU+Hgq3X9KE7bKh4WkCem24Np4uA+irhpvZmIoHyINsYW46lGfwJvM8= kapandya@localhost.localdomain", "key_options": null, "keyfile": "/home/kapandya/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kapandya", "validate_certs": true} 2021-11-11T20:39:13.224 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'prasad', 'ovpn': 'prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "prasad", "ovpn": "prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNEEz49M5KdLcp0cbKrE9iihrH5PEhsPgnAxac+aQwvnTstOOlUOyR+TvHiRa7sgXHuFX/UXo+k3Qmhs5U/sMZWZmmN3Jb3fKT78lSBbGnByclfPoGVTTLOHxwo+RFE79Sp3e30oshJ3DwqYgiX6JjM8B2Y7d5Tv0Ss5uEZswvmCEjdTLDSsoxrsOjVxBpuArOfCxes8PnKDdq21mWldpP66iwFel2GgOGSc85iVHKcOE2OpVMazMzXoMSMFoOpCULKjFfXBuDIqkXwJmtLJaiCgcVmTR56KXh7JpZ7vAeJ9ttv9qgOckqKdI3jEjeDT+TrdLlRWghBE7FRkm0vmun prasad.krishnan@macprasad", "key_options": null, "keyfile": "/home/prasad/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "prasad", "validate_certs": true} 2021-11-11T20:39:13.226 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'prasad', 'ovpn': 'prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "prasad", "ovpn": "prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNEEz49M5KdLcp0cbKrE9iihrH5PEhsPgnAxac+aQwvnTstOOlUOyR+TvHiRa7sgXHuFX/UXo+k3Qmhs5U/sMZWZmmN3Jb3fKT78lSBbGnByclfPoGVTTLOHxwo+RFE79Sp3e30oshJ3DwqYgiX6JjM8B2Y7d5Tv0Ss5uEZswvmCEjdTLDSsoxrsOjVxBpuArOfCxes8PnKDdq21mWldpP66iwFel2GgOGSc85iVHKcOE2OpVMazMzXoMSMFoOpCULKjFfXBuDIqkXwJmtLJaiCgcVmTR56KXh7JpZ7vAeJ9ttv9qgOckqKdI3jEjeDT+TrdLlRWghBE7FRkm0vmun prasad.krishnan@macprasad", "key_options": null, "keyfile": "/home/prasad/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "prasad", "validate_certs": true} 2021-11-11T20:39:13.453 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vikhyat', 'ovpn': 'vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vikhyat", "ovpn": "vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCu9CW99ySPXSKQpLbefu1A0KqbUQIzvUOJizIu4azEgdlOqUnOMR/GC6jgOdiVL5mNb/U5bg7yQuEHgPAtmPRotvbI6/FSwIAR9+GgiuKSOJCa4s7QzWbHwv9g7wS2SLURw8UERSzlLlT8KQyGj+GkBhToWp4PTXC7te61c8XJNGF6giLmIAky4SH8uPI1zeLwwviJQomjFWZZ1RTbcRJxvGPjPtuUKDlF7QgXTLE9ZgsNopusC6PfRd7N8Udkl2OYPAzA+Jun0IRnQyjRXG3c2sYXQR8YYGICsxvnm4l81vKdxRL34K7LqBUHQXUcoCjJy7+/2/1dca/AMHgi4sZwgeYQ/U9jZg5jbm7o4MpDUFsS8W6OO/ZPiuYTswGBNRXhPmbGFnfa9dBXBS0p+buHF8PcTSLH0Dp2nhHkK1ehGmBFVPw6nPj3wXA9BKuNc+g7AiNUyRKyCWroFJ0VwUQGbI6ZMccO2cI7QpYmIqPCl3am/vUSNq4j0m/iURR7+t0= vikhyat@redhat", "key_options": null, "keyfile": "/home/vikhyat/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vikhyat", "validate_certs": true} 2021-11-11T20:39:13.455 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vikhyat', 'ovpn': 'vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vikhyat", "ovpn": "vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCu9CW99ySPXSKQpLbefu1A0KqbUQIzvUOJizIu4azEgdlOqUnOMR/GC6jgOdiVL5mNb/U5bg7yQuEHgPAtmPRotvbI6/FSwIAR9+GgiuKSOJCa4s7QzWbHwv9g7wS2SLURw8UERSzlLlT8KQyGj+GkBhToWp4PTXC7te61c8XJNGF6giLmIAky4SH8uPI1zeLwwviJQomjFWZZ1RTbcRJxvGPjPtuUKDlF7QgXTLE9ZgsNopusC6PfRd7N8Udkl2OYPAzA+Jun0IRnQyjRXG3c2sYXQR8YYGICsxvnm4l81vKdxRL34K7LqBUHQXUcoCjJy7+/2/1dca/AMHgi4sZwgeYQ/U9jZg5jbm7o4MpDUFsS8W6OO/ZPiuYTswGBNRXhPmbGFnfa9dBXBS0p+buHF8PcTSLH0Dp2nhHkK1ehGmBFVPw6nPj3wXA9BKuNc+g7AiNUyRKyCWroFJ0VwUQGbI6ZMccO2cI7QpYmIqPCl3am/vUSNq4j0m/iURR7+t0= vikhyat@redhat", "key_options": null, "keyfile": "/home/vikhyat/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vikhyat", "validate_certs": true} 2021-11-11T20:39:13.682 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'shraddhaag', 'ovpn': 'shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "shraddhaag", "ovpn": "shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCipD0d1tI/Cff3AeOZTQ9eb6pZ/NOCAOw7MZG/JgpCs8+T3WqiUEYefsb2PuSa85vV2GC7kq0M9ux7+U+vZZf05geYiU2S9eTucLEu2+YE+RaAvIK+ZB134/33JSPGIVubXgGWZw/ZFCEYXmYk3kyEg1YDIk/BEaoG7PsDiSB4j0W7/WiV2bEEdSlTXt5C1/kH0qn4H9wQMiBSpdMgdaRWw60HjObDd6LASeUVgabbyclypravZdpYBnits4ayoN8VG1aZe4Qhm1SzP/CfxAAYfxo9NbqSBWJmn3iSN8UKHhCFrRCTaYic3z3usr8784VsA7z/LxK0fs6xjhoxRTa3 shraddha.agrawal000@gmail.com", "key_options": null, "keyfile": "/home/shraddhaag/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "shraddhaag", "validate_certs": true} 2021-11-11T20:39:13.684 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'shraddhaag', 'ovpn': 'shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "shraddhaag", "ovpn": "shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCipD0d1tI/Cff3AeOZTQ9eb6pZ/NOCAOw7MZG/JgpCs8+T3WqiUEYefsb2PuSa85vV2GC7kq0M9ux7+U+vZZf05geYiU2S9eTucLEu2+YE+RaAvIK+ZB134/33JSPGIVubXgGWZw/ZFCEYXmYk3kyEg1YDIk/BEaoG7PsDiSB4j0W7/WiV2bEEdSlTXt5C1/kH0qn4H9wQMiBSpdMgdaRWw60HjObDd6LASeUVgabbyclypravZdpYBnits4ayoN8VG1aZe4Qhm1SzP/CfxAAYfxo9NbqSBWJmn3iSN8UKHhCFrRCTaYic3z3usr8784VsA7z/LxK0fs6xjhoxRTa3 shraddha.agrawal000@gmail.com", "key_options": null, "keyfile": "/home/shraddhaag/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "shraddhaag", "validate_certs": true} 2021-11-11T20:39:13.911 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'gkyratsas', 'ovpn': 'gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "gkyratsas", "ovpn": "gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCpE5V9JG1WhkHSYqjqaijBb1MLyjcKQlmj5WQ8w5832n6x9J4yDO8wI4qmiX3XeiXPlmW6XjAq5keyukrQe/yVI6NDXzgRxaHDmNj1AR73gyUVqRXpJprrqqsnLHawIRovrBD/HNhTv/2G8xG3BHakac1+qj7tZe4Jjx0aG1HbRhhJ5Q98EgvTkDAhktnlvF9OXXC1WI7QirHx0pRHa40SYyFLfOQDbQSCzb1TAAY5NGn3SwK3s2mWeoEsAJpCLrkNfSTbR3Z0mylAJUn+PayqhJegjwKmLaONjx8ZHXN970JHgojBBGOD30J/NnJQ7N6eRl+ekjpHmLAYGbe2oCwD gkyratsas@suse.com", "key_options": null, "keyfile": "/home/gkyratsas/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "gkyratsas", "validate_certs": true} 2021-11-11T20:39:13.922 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'gkyratsas', 'ovpn': 'gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "gkyratsas", "ovpn": "gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCpE5V9JG1WhkHSYqjqaijBb1MLyjcKQlmj5WQ8w5832n6x9J4yDO8wI4qmiX3XeiXPlmW6XjAq5keyukrQe/yVI6NDXzgRxaHDmNj1AR73gyUVqRXpJprrqqsnLHawIRovrBD/HNhTv/2G8xG3BHakac1+qj7tZe4Jjx0aG1HbRhhJ5Q98EgvTkDAhktnlvF9OXXC1WI7QirHx0pRHa40SYyFLfOQDbQSCzb1TAAY5NGn3SwK3s2mWeoEsAJpCLrkNfSTbR3Z0mylAJUn+PayqhJegjwKmLaONjx8ZHXN970JHgojBBGOD30J/NnJQ7N6eRl+ekjpHmLAYGbe2oCwD gkyratsas@suse.com", "key_options": null, "keyfile": "/home/gkyratsas/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "gkyratsas", "validate_certs": true} 2021-11-11T20:39:14.130 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'prateek', 'ovpn': 'prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "prateek", "ovpn": "prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC9sZe9p48s6RSpluK1R97X9tsN3tWiidPbdyXvyMUXifYQb3nth9wMWUakrVBp+QGEkqALLIf3so3MfgbRvYexzNfHvRT7Lxn8D/isiCpWa4aLM/0alprV+RI38dR0mI3IVb3aIEYmgp2IY4yzybWTHCmS1VUrUJJ0pOfXOssq0xgo3qeaSQBHYRKasqR8GWAD5MWpG6/kg6gxI7o0aHVVTXy5ADB7NS1LEZr06xxNm4exeYvl9TzXrbGZWf2gC8ZpXhmFHI80A+TKEZRU6wmcju7G69nyEEF8/ebcBKmZB4eTsDN7qR7lc9dxKxiiZcJXc/MWy4bSu69NhOeuq2xx7bdexS8OHKdgYH9gsJolu5zgN20IHzaRgswO3XIe3nh2Ud8wntwBCwmJ9bf5YByITGjpOhntkvWAP3AXkQ1B4HXCdlUNW+4pMexDe2D05LHOwxcS8RVIH6ELu8rWe/QBA+DFcyZz4xE55pbfVt3FCEFrAxUn5qkyzszwPMjtK4XAnn9kmnw2B0k6XLZOZd+n7W8ege6wih+gwfDhYMZX0qzi3HdAK7nk5vEOFibCztLaogzfc6fdFRx8OIWJQXqx5I1641FU3UE6sVxku4IMDTq2GVyUWKdSN6y8gBiDPVe3xu4qM4xghSnaAmfB45IrcNvibBjikNva2FHt8lcutQ== prateek@prateek-Dell", "key_options": null, "keyfile": "/home/prateek/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "prateek", "validate_certs": true} 2021-11-11T20:39:14.141 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'prateek', 'ovpn': 'prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "prateek", "ovpn": "prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC9sZe9p48s6RSpluK1R97X9tsN3tWiidPbdyXvyMUXifYQb3nth9wMWUakrVBp+QGEkqALLIf3so3MfgbRvYexzNfHvRT7Lxn8D/isiCpWa4aLM/0alprV+RI38dR0mI3IVb3aIEYmgp2IY4yzybWTHCmS1VUrUJJ0pOfXOssq0xgo3qeaSQBHYRKasqR8GWAD5MWpG6/kg6gxI7o0aHVVTXy5ADB7NS1LEZr06xxNm4exeYvl9TzXrbGZWf2gC8ZpXhmFHI80A+TKEZRU6wmcju7G69nyEEF8/ebcBKmZB4eTsDN7qR7lc9dxKxiiZcJXc/MWy4bSu69NhOeuq2xx7bdexS8OHKdgYH9gsJolu5zgN20IHzaRgswO3XIe3nh2Ud8wntwBCwmJ9bf5YByITGjpOhntkvWAP3AXkQ1B4HXCdlUNW+4pMexDe2D05LHOwxcS8RVIH6ELu8rWe/QBA+DFcyZz4xE55pbfVt3FCEFrAxUn5qkyzszwPMjtK4XAnn9kmnw2B0k6XLZOZd+n7W8ege6wih+gwfDhYMZX0qzi3HdAK7nk5vEOFibCztLaogzfc6fdFRx8OIWJQXqx5I1641FU3UE6sVxku4IMDTq2GVyUWKdSN6y8gBiDPVe3xu4qM4xghSnaAmfB45IrcNvibBjikNva2FHt8lcutQ== prateek@prateek-Dell", "key_options": null, "keyfile": "/home/prateek/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "prateek", "validate_certs": true} 2021-11-11T20:39:14.369 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jelopez', 'ovpn': 'jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jelopez", "ovpn": "jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDFmmTp8+YDXU27xnSPxhWwouslpkvp1eULVp63taoXceDOnH8EMWv0wiEjN/P+Fcf/UB+iU8bELyHubZiqnOUr9f2pGX4jkJ6Yty1p5BUK4my7Si5J0/JawEDVQm4HpFWMKUVKYumvz3UFuBoINHVTqr1cLIDAt7mdeKfJqFB45AqxDUIPP4d8iznNCw+QO2CMxk0o/wMNHbygZZrHAiDngXQxwhe6oR17RfolhREy0JjHgojRjxgagpG7O8WW3OPI5Outnz5Pvraas5q7w+b3xbB9vdTObW/Nii5VOpTdIsFzBvp46hLoRSIcB8pRLUrxPJBPOY7G0zbopDRLB2zLNGKIH2zxEHJOeY928uUbWT22b9mGu/L95FFY7hmeRAicCUeixMTgXgczeQlHDn5eqQzLI3PNP7JX/0dauCD27GRKsDUnpIDj9RqgCQDVueIQQMC6QMa9dYQmps0r9Ljp8R84WjYFRP9oOI+POOzbgmRO0JCVnPqz9MKyGVby9/WFBHcvG6xmwACez00oqe62z+rhcavGXViX5CxGMV6vUTXugBa4ZkXlzs+djzYWfZ33ZKVeRIwj2Pu1PRTGzB8e2BaKqJMN0FVtzpV4hom4Y/9vIPyDpjRjNuKzrkYW1M8PxBxn+EJUPMLhr3jFMCo6HSxHoFXy8BzZoFkZWqap3w== JCL@IT-MINI-JCL.local", "key_options": null, "keyfile": "/home/jelopez/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jelopez", "validate_certs": true} 2021-11-11T20:39:14.371 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jelopez', 'ovpn': 'jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jelopez", "ovpn": "jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDFmmTp8+YDXU27xnSPxhWwouslpkvp1eULVp63taoXceDOnH8EMWv0wiEjN/P+Fcf/UB+iU8bELyHubZiqnOUr9f2pGX4jkJ6Yty1p5BUK4my7Si5J0/JawEDVQm4HpFWMKUVKYumvz3UFuBoINHVTqr1cLIDAt7mdeKfJqFB45AqxDUIPP4d8iznNCw+QO2CMxk0o/wMNHbygZZrHAiDngXQxwhe6oR17RfolhREy0JjHgojRjxgagpG7O8WW3OPI5Outnz5Pvraas5q7w+b3xbB9vdTObW/Nii5VOpTdIsFzBvp46hLoRSIcB8pRLUrxPJBPOY7G0zbopDRLB2zLNGKIH2zxEHJOeY928uUbWT22b9mGu/L95FFY7hmeRAicCUeixMTgXgczeQlHDn5eqQzLI3PNP7JX/0dauCD27GRKsDUnpIDj9RqgCQDVueIQQMC6QMa9dYQmps0r9Ljp8R84WjYFRP9oOI+POOzbgmRO0JCVnPqz9MKyGVby9/WFBHcvG6xmwACez00oqe62z+rhcavGXViX5CxGMV6vUTXugBa4ZkXlzs+djzYWfZ33ZKVeRIwj2Pu1PRTGzB8e2BaKqJMN0FVtzpV4hom4Y/9vIPyDpjRjNuKzrkYW1M8PxBxn+EJUPMLhr3jFMCo6HSxHoFXy8BzZoFkZWqap3w== JCL@IT-MINI-JCL.local", "key_options": null, "keyfile": "/home/jelopez/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jelopez", "validate_certs": true} 2021-11-11T20:39:14.609 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dcain', 'ovpn': 'dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dcain", "ovpn": "dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCdbJDYY43Zu0EKAGgrSKEobyXIxdhT0AeTT3yBoNY+YdYDScimJsQzK4hKNJIn1xC3klr0dSnsqeJru+SA4GMhbdcsmVyudhutCw75R/vFoRHOwaeVS2SMiBAtDbAYGQ3dh6LapkmXeLURcycNbebUV4HM9XsYqnuL8J2XyY4tRKgHrAd6OQw4jsXDOxwTRlHM822N/2kgv8iEwiXAJoUoymbr8mnfSxuuaH0oiKBwuaK1wuaDrlJyXkXNslPq89/1f8hJfjixY2dQdW7Xk5m3t5ptQxWENpw79OMRJ4cDW4G7LrtFONom5KQ+9TBZJIhNgJobkqI4qwinrWF8qMW5 dcain@fullsteam", "key_options": null, "keyfile": "/home/dcain/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dcain", "validate_certs": true} 2021-11-11T20:39:14.611 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dcain', 'ovpn': 'dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dcain", "ovpn": "dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCdbJDYY43Zu0EKAGgrSKEobyXIxdhT0AeTT3yBoNY+YdYDScimJsQzK4hKNJIn1xC3klr0dSnsqeJru+SA4GMhbdcsmVyudhutCw75R/vFoRHOwaeVS2SMiBAtDbAYGQ3dh6LapkmXeLURcycNbebUV4HM9XsYqnuL8J2XyY4tRKgHrAd6OQw4jsXDOxwTRlHM822N/2kgv8iEwiXAJoUoymbr8mnfSxuuaH0oiKBwuaK1wuaDrlJyXkXNslPq89/1f8hJfjixY2dQdW7Xk5m3t5ptQxWENpw79OMRJ4cDW4G7LrtFONom5KQ+9TBZJIhNgJobkqI4qwinrWF8qMW5 dcain@fullsteam", "key_options": null, "keyfile": "/home/dcain/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dcain", "validate_certs": true} 2021-11-11T20:39:14.838 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'sunny', 'ovpn': 'sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sunny", "ovpn": "sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDs2+6XwRPdiDHT+63Yab00VBulg96/OLEORtNUvm7TDRMpcx4OD2oOnbsE4rG3dOdXNgT17H15c6MWFX2a2uC2h1j4hJTa+dG567HrYXRHG60Kwg5Sviiuq/ETPoH/r3OSvHs0HJ4Y9qYPQ5j0k7gG1XgIY3QHDWNRsXiyRXP9FEfoaOKABDTUoLPjqRX+4ExJd1Nz6DLxqIlre8wfqF5L1l12vec8JU3HB5NSRa87HxKFtLLIAtw7B9H/HggqX5wWVEnXq2qN4o6OOePhaO5cjWAZxOtT+AJTyHkeyaWLwIGOHBVZS5BKti6jLc+PxQxAflYlu9X3xh/2FlA+9H+ABZwMHE3ruUfTVbviYBbtccMCvkzcvFgnvaUCHjd7Iv/T8/JHNfBgFgigcX0Vei0+PZQ5oC5b7YpKT241NA8DCGZNgr1KuL9LazkvbkbpDcaHUOrkWsmSW1RRMC6ffYkbAI9QiBJFpImDAWP1u+1yCGhzPKBfGNt91GcLCD6ML90= sunny@devBox", "key_options": null, "keyfile": "/home/sunny/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sunny", "validate_certs": true} 2021-11-11T20:39:14.840 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'sunny', 'ovpn': 'sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sunny", "ovpn": "sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDs2+6XwRPdiDHT+63Yab00VBulg96/OLEORtNUvm7TDRMpcx4OD2oOnbsE4rG3dOdXNgT17H15c6MWFX2a2uC2h1j4hJTa+dG567HrYXRHG60Kwg5Sviiuq/ETPoH/r3OSvHs0HJ4Y9qYPQ5j0k7gG1XgIY3QHDWNRsXiyRXP9FEfoaOKABDTUoLPjqRX+4ExJd1Nz6DLxqIlre8wfqF5L1l12vec8JU3HB5NSRa87HxKFtLLIAtw7B9H/HggqX5wWVEnXq2qN4o6OOePhaO5cjWAZxOtT+AJTyHkeyaWLwIGOHBVZS5BKti6jLc+PxQxAflYlu9X3xh/2FlA+9H+ABZwMHE3ruUfTVbviYBbtccMCvkzcvFgnvaUCHjd7Iv/T8/JHNfBgFgigcX0Vei0+PZQ5oC5b7YpKT241NA8DCGZNgr1KuL9LazkvbkbpDcaHUOrkWsmSW1RRMC6ffYkbAI9QiBJFpImDAWP1u+1yCGhzPKBfGNt91GcLCD6ML90= sunny@devBox", "key_options": null, "keyfile": "/home/sunny/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sunny", "validate_certs": true} 2021-11-11T20:39:15.066 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'xuxuehan', 'ovpn': 'xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "xuxuehan", "ovpn": "xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4pIcNiZKVIfT9GxyTe/B8G2GyHKxrAle3iExEzLH5bBCKtBrottX1y66KBzQSsrAb0z9h+A+uxlswlY6VM/gNos9VRB6x4NRB4ADY1Bku+JsKs5aXwRB1zz2ueZhrFN9hJwMl1heLICx85ksHBdm0q+JPdUAV+/LBdr0H/IjWveV34WL3Awf7QikzyKhzUciMB2gN9PJXTW0bl8PP56y60ys++L3nMSRJ+15uPcWl8MS3D2vABII0Tu+3PsxzrTTNodElyQ6G/fwyFfWhdp7fkidjR7mDDx8NyWfCl2rKejXv9LnVC0wb3tFoQqvwdz6shN4E51EIEGXHcLUeRpBhL2pFOqIZ0qfnmLIQVZ88RKMTgo42CgtmS5UzxCLOyKOh9GxF7DDEajTinJFJJVbNJ5xF494H2geSnmP1o1AecGsGbCZ4ndhS9unSfIZLAyv1J3wZlAlMsvxXgY7yrh1srUiASVyKK8jba1IT+v3TXR8kQUCNsLKKbvuX/l0IXME= xuxuehan@bogon\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBd4eSizc2zAkWDy3mFZUhELO2wXDgkVxDNM/Nw2zT1li3L+6YOjFfdT4NOxpkg5laLd54bdKb5GVRA6iKqAO+aqo53xHHJGC8OY5eJaUPjtpypgIWO4Qzbos2ZzsC+nX5jvi8O9nM+JkabSW5iFv3gGPTqHixFYkxK2NIBa/UtXFOhmb1wZ6CG5Q+WObCosBUEMWokC8ntCADfhi6qDsO2fol6mYUhYEE/0nhQ5uLRrVIM5QM5dhrQVEeAs4KcYpmPDw6q6AKgx/f5zObh5OSJFTVsgx2cqzdQVJVDDx2OD2QVRtZ9fgvI2EGmKSwe1TUQOdqa5bmsVIp2Jfi8MC1 xuxuehan@localhost.localdomain\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCwIokXSvPYP3TRlSw0SBgmMFW5jVltGf4c6CqjIW4KKCgsL4c4Vs0FztRyDGc9p0krZUoGJCZHErVToXm6/lSq8JlfrlLX4X4yh37fUifWqc2UTJGeH8kLaP3WEt9vT8eQ9h8t2GqlMlGQU8KzVAdC+c5IwrbGRsS6S7D09/y0rklkhdJ6nuj8cssN/2TzLF9YIvVcFZHdyznk84eB7TgpPKD88MOpTnEgWFlD4vAWEuHpkuxgj1KI9LscgSFcW9/RxLeRxcV0GypZ8jPmczoCnin/ZOstCTpbcmjL 2021-11-11T20:39:15.068 INFO:teuthology.task.ansible.out:d+LW1kgGXXQSkf09iHeJ/cyCw6ooZOkynVwwmI9Uz2ZaMedCUBpG0bKclIerJ9jn+CoTs3U3Orhd9+6MVtCHX9VNJbVa4q0KCS1sp5S3HDrYkgUyawUxdYorrLZcqfRHWWo1luiridmuQdr8lQ6dvNpz4rhsCrAG6S1eeaC/s+o0Rzpv1YxU64JzquZ6WNR+W3U6Bg3iYkSoDCK9byf0rsptyL8= luqixi@luqixideMacBook-Air.local", "key_options": null, "keyfile": "/home/xuxuehan/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "xuxuehan", "validate_certs": true} ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'xuxuehan', 'ovpn': 'xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "xuxuehan", "ovpn": "xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4pIcNiZKVIfT9GxyTe/B8G2GyHKxrAle3iExEzLH5bBCKtBrottX1y66KBzQSsrAb0z9h+A+uxlswlY6VM/gNos9VRB6x4NRB4ADY1Bku+JsKs5aXwRB1zz2ueZhrFN9hJwMl1heLICx85ksHBdm0q+JPdUAV+/LBdr0H/IjWveV34WL3Awf7QikzyKhzUciMB2gN9PJXTW0bl8PP56y60ys++L3nMSRJ+15uPcWl8MS3D2vABII0Tu+3PsxzrTTNodElyQ6G/fwyFfWhdp7fkidjR7mDDx8NyWfCl2rKejXv9LnVC0wb3tFoQqvwdz6shN4E51EIEGXHcLUeRpBhL2pFOqIZ0qfnmLIQVZ88RKMTgo42CgtmS5UzxCLOyKOh9GxF7DDEajTinJFJJVbNJ5xF494H2geSnmP1o1AecGsGbCZ4ndhS9unSfIZLAyv1J3wZlAlMsvxXgY7yrh1srUiASVyKK8jba1IT+v3TXR8kQUCNsLKKbvuX/l0IXME= xuxuehan@bogon\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBd4eSizc2zAkWDy3mFZUhELO2wXDgkVxDNM/Nw2zT1li3L+6YOjFfdT4NOxpkg5laLd54bdKb5GVRA6iKqAO+aqo53xHHJGC8OY5eJaUPjtpypgIWO4Qzbos2ZzsC+nX5jvi8O9nM+JkabSW5iFv3gGPTqHixFYkxK2NIBa/UtXFOhmb1wZ6CG5Q+WObCosBUEMWokC8ntCADfhi6qDsO2fol6mYUhYEE/0nhQ5uLRrVIM5QM5dhrQVEeAs4KcYpmPDw6q6AKgx/f5zObh5OSJFT 2021-11-11T20:39:15.070 INFO:teuthology.task.ansible.out:Vsgx2cqzdQVJVDDx2OD2QVRtZ9fgvI2EGmKSwe1TUQOdqa5bmsVIp2Jfi8MC1 xuxuehan@localhost.localdomain\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCwIokXSvPYP3TRlSw0SBgmMFW5jVltGf4c6CqjIW4KKCgsL4c4Vs0FztRyDGc9p0krZUoGJCZHErVToXm6/lSq8JlfrlLX4X4yh37fUifWqc2UTJGeH8kLaP3WEt9vT8eQ9h8t2GqlMlGQU8KzVAdC+c5IwrbGRsS6S7D09/y0rklkhdJ6nuj8cssN/2TzLF9YIvVcFZHdyznk84eB7TgpPKD88MOpTnEgWFlD4vAWEuHpkuxgj1KI9LscgSFcW9/RxLeRxcV0GypZ8jPmczoCnin/ZOstCTpbcmjLd+LW1kgGXXQSkf09iHeJ/cyCw6ooZOkynVwwmI9Uz2ZaMedCUBpG0bKclIerJ9jn+CoTs3U3Orhd9+6MVtCHX9VNJbVa4q0KCS1sp5S3HDrYkgUyawUxdYorrLZcqfRHWWo1luiridmuQdr8lQ6dvNpz4rhsCrAG6S1eeaC/s+o0Rzpv1YxU64JzquZ6WNR+W3U6Bg3iYkSoDCK9byf0rsptyL8= luqixi@luqixideMacBook-Air.local", "key_options": null, "keyfile": "/home/xuxuehan/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "xuxuehan", "validate_certs": true} 2021-11-11T20:39:15.296 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'aclewett', 'ovpn': 'aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "aclewett", "ovpn": "aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDMOuxxTZyxz/TEVA+ODgtLrNcSsUhKE64BZg/Ub96LLffQXQc9SYT/0YI4wxu2vSsky05PnVSON7jqGxunCt0+AdRqxXhW4ttmRIOz8rq8SiMnnLayV34nIF1G0aYTkgY4ej4+VmSVIHU3bb5mp/Q3D0hxQwDPdgOJWNyW9oYerHbnx6lVgchvsZHB9VO09GNHcAsyUz4p+xBf9iR7eS74p4dZmUbth/oBBqpOa4NgNr69iyZ9a/pNDLxaVQoIpDFrevk5UU5nUHQq2frFNxF/Y2Ja1EBXCavvPF/m5v3J3xNEu3SI/GbLFCPKCKeBDpXJbe1QqTBKJjMl9qGmKEJ3 aclewett@mac", "key_options": null, "keyfile": "/home/aclewett/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "aclewett", "validate_certs": true} 2021-11-11T20:39:15.307 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'aclewett', 'ovpn': 'aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "aclewett", "ovpn": "aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDMOuxxTZyxz/TEVA+ODgtLrNcSsUhKE64BZg/Ub96LLffQXQc9SYT/0YI4wxu2vSsky05PnVSON7jqGxunCt0+AdRqxXhW4ttmRIOz8rq8SiMnnLayV34nIF1G0aYTkgY4ej4+VmSVIHU3bb5mp/Q3D0hxQwDPdgOJWNyW9oYerHbnx6lVgchvsZHB9VO09GNHcAsyUz4p+xBf9iR7eS74p4dZmUbth/oBBqpOa4NgNr69iyZ9a/pNDLxaVQoIpDFrevk5UU5nUHQq2frFNxF/Y2Ja1EBXCavvPF/m5v3J3xNEu3SI/GbLFCPKCKeBDpXJbe1QqTBKJjMl9qGmKEJ3 aclewett@mac", "key_options": null, "keyfile": "/home/aclewett/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "aclewett", "validate_certs": true} 2021-11-11T20:39:15.535 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'cblum', 'ovpn': 'cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "cblum", "ovpn": "cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCdTLDprr3DXtsgiqrB/miglJwSkY7BAv+OW9sC0ZHQf8Y4dFoUtr7q9fd6W7ehKL6aMjya/8eQSMPRoEIZFD8f2KPCuE74Wgwgv2cpAkFmLjcPtjixqLo3LGFCyEJWGLQxjlthdzx59mpCWgBsEJQmlKP36JIYjILIMtXtsETIUGFa8QPIn9sMdIgutPPf3yB/6DHcR7C6yIAuvbJactEsiCt4C8+uJinvCii3PzYOMAEd+mictPGk8nOCQFZrI+ZmwTz4UbJylxFqnX8lhhoPIUEfNAXlb3lkZWH1LaFOxarbmupuPYcg2rT0+kFtyNr3/mAx0FO01jADYooqcW1Z cardno:000604936806", "key_options": null, "keyfile": "/home/cblum/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "cblum", "validate_certs": true} 2021-11-11T20:39:15.537 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'cblum', 'ovpn': 'cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "cblum", "ovpn": "cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCdTLDprr3DXtsgiqrB/miglJwSkY7BAv+OW9sC0ZHQf8Y4dFoUtr7q9fd6W7ehKL6aMjya/8eQSMPRoEIZFD8f2KPCuE74Wgwgv2cpAkFmLjcPtjixqLo3LGFCyEJWGLQxjlthdzx59mpCWgBsEJQmlKP36JIYjILIMtXtsETIUGFa8QPIn9sMdIgutPPf3yB/6DHcR7C6yIAuvbJactEsiCt4C8+uJinvCii3PzYOMAEd+mictPGk8nOCQFZrI+ZmwTz4UbJylxFqnX8lhhoPIUEfNAXlb3lkZWH1LaFOxarbmupuPYcg2rT0+kFtyNr3/mAx0FO01jADYooqcW1Z cardno:000604936806", "key_options": null, "keyfile": "/home/cblum/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "cblum", "validate_certs": true} 2021-11-11T20:39:15.763 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mlatimer', 'ovpn': 'mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mlatimer", "ovpn": "mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDPUCIVfNQOOG+uAeKWo7tLAdgGO/oQu4exTE3XLfyTVdd8u6pbQCmIDFEnbGgMtlYuWJlCJtxJIzEZJ5HIwf+po3y7kb7HxEeWnEb9DsqJB4nCRE8fHnNLgF3bh9gvE7sOZdNXEKqtsnGcaaS81oXQ6QgkFGQzlOBOqxj7jIvad7AtNWD3WbJi/uTVDsWcukU51tn/aEW+hHMD7Y5QJkLH7O0PrCeEZCUCWoiMP7sNwACBym0DsqX+POkGJOaGx9EeNhGVA4B6o3xYXJPqmFbgMWMfG3/Y3SiheZjxzySMQ7FKW4Q817NaWqbjtLZQsSZw7JNqNFkcdTzVZwwCOJ1AAecajpniKeQLnyiWQ3Z/EkSGGwtDj67wo7uqT85tpOyLfYhQm6Zp+xSh+I6s0YJ0xO/huLKzeT5D1r5vVMk2VdcLD3CCxXUwZgaMUxYAC2Wq6mpIkxVZFQLbGuA/dWkQbFsrpbf7ES+kh+uzp4R6dQ29OjXbVcSfF2vyIuA+VUKoyj370YOZQhs50yuAJ4BSHlEgoJdRzi3XUX48h45XHiZcCY3X8LTX6a+0KAIgqeAWdFhRgQtAxoRYjhOxC3k/boBc8sEQ3o4pkxnhtE5Y/tnM2CPkPMX9HWYirNf9GyZTYcR/5rDf4ETQ18G68WLaNK758OuUPjSkwttcyLd4dw== mlatimer@suse.com", "key_options": null, "keyfile": "/home/mlatimer/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mlatimer", "validate_certs": true} 2021-11-11T20:39:15.765 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mlatimer', 'ovpn': 'mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mlatimer", "ovpn": "mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDPUCIVfNQOOG+uAeKWo7tLAdgGO/oQu4exTE3XLfyTVdd8u6pbQCmIDFEnbGgMtlYuWJlCJtxJIzEZJ5HIwf+po3y7kb7HxEeWnEb9DsqJB4nCRE8fHnNLgF3bh9gvE7sOZdNXEKqtsnGcaaS81oXQ6QgkFGQzlOBOqxj7jIvad7AtNWD3WbJi/uTVDsWcukU51tn/aEW+hHMD7Y5QJkLH7O0PrCeEZCUCWoiMP7sNwACBym0DsqX+POkGJOaGx9EeNhGVA4B6o3xYXJPqmFbgMWMfG3/Y3SiheZjxzySMQ7FKW4Q817NaWqbjtLZQsSZw7JNqNFkcdTzVZwwCOJ1AAecajpniKeQLnyiWQ3Z/EkSGGwtDj67wo7uqT85tpOyLfYhQm6Zp+xSh+I6s0YJ0xO/huLKzeT5D1r5vVMk2VdcLD3CCxXUwZgaMUxYAC2Wq6mpIkxVZFQLbGuA/dWkQbFsrpbf7ES+kh+uzp4R6dQ29OjXbVcSfF2vyIuA+VUKoyj370YOZQhs50yuAJ4BSHlEgoJdRzi3XUX48h45XHiZcCY3X8LTX6a+0KAIgqeAWdFhRgQtAxoRYjhOxC3k/boBc8sEQ3o4pkxnhtE5Y/tnM2CPkPMX9HWYirNf9GyZTYcR/5rDf4ETQ18G68WLaNK758OuUPjSkwttcyLd4dw== mlatimer@suse.com", "key_options": null, "keyfile": "/home/mlatimer/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mlatimer", "validate_certs": true} 2021-11-11T20:39:15.991 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'shyamsundarr', 'ovpn': 'shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "shyamsundarr", "ovpn": "shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCYh28a9R3a7jpRfPqVAL9EcD+S6KkZzKTfCmRn4s8Fg39kUZsgnWUZQ3jUxKSeBA00te7WMR8CsIEzulm7f+/egNuPcvzNI/SaaCDtVMaxqlEuW85ll8uBQi3s4SLmOu9oS+5oid4cqTrGTcG6uhl29oWS/53SxoGh+T+y6GesO2cYRuTpcUHxGMvIgZe/bKtVGdP1y7rQd62e8emb9UupMBxaHpkYj9AaMuLjovHbBgecWbe4PlzPWNzl+eITVk2d6CxxIem4/bw9VHkRjQZlhL2nbwRgtczLYhlaZAd4ePEtsqA+7nTDjTDhXXHVIf1pDAsJbZEdgQCSGH3FzxtrQ2hZJrH/NIKdHk/OtI06e7POSGj7PcHzTvMjQSFnTpJXinxwzn0lvFSlmABN/00lIE0p2ZXfPs6dgWJ17JnTq6dMUoTiGhfKphPHyfSXJCzfa2VB+lQQ5eAaa9cRXy9pLmWGHXVGz2hsBMtyYmTjR/wr8kdTzeK+Jk4opBtGgcxDustg59b9x/jvgoc3HCdtnkqLTug2KmutB6kb+4KnVO07GsSp0RmNtDmCRcXs9YN/eia9ICtIGDWPc3YDRppsw1Vktza/+ZW5vGzIKILZTo4SNGcg19zQw4Qxd9eLkXfB66vE2CQvA90CGo7mcye+UWjjXLr7i0cDEszAloQ9aQ== srangana@redhat.com", "key_options": null, "keyfile": "/home/shyamsundarr/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "shyamsundarr", "validate_certs": true} 2021-11-11T20:39:15.993 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'shyamsundarr', 'ovpn': 'shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "shyamsundarr", "ovpn": "shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCYh28a9R3a7jpRfPqVAL9EcD+S6KkZzKTfCmRn4s8Fg39kUZsgnWUZQ3jUxKSeBA00te7WMR8CsIEzulm7f+/egNuPcvzNI/SaaCDtVMaxqlEuW85ll8uBQi3s4SLmOu9oS+5oid4cqTrGTcG6uhl29oWS/53SxoGh+T+y6GesO2cYRuTpcUHxGMvIgZe/bKtVGdP1y7rQd62e8emb9UupMBxaHpkYj9AaMuLjovHbBgecWbe4PlzPWNzl+eITVk2d6CxxIem4/bw9VHkRjQZlhL2nbwRgtczLYhlaZAd4ePEtsqA+7nTDjTDhXXHVIf1pDAsJbZEdgQCSGH3FzxtrQ2hZJrH/NIKdHk/OtI06e7POSGj7PcHzTvMjQSFnTpJXinxwzn0lvFSlmABN/00lIE0p2ZXfPs6dgWJ17JnTq6dMUoTiGhfKphPHyfSXJCzfa2VB+lQQ5eAaa9cRXy9pLmWGHXVGz2hsBMtyYmTjR/wr8kdTzeK+Jk4opBtGgcxDustg59b9x/jvgoc3HCdtnkqLTug2KmutB6kb+4KnVO07GsSp0RmNtDmCRcXs9YN/eia9ICtIGDWPc3YDRppsw1Vktza/+ZW5vGzIKILZTo4SNGcg19zQw4Qxd9eLkXfB66vE2CQvA90CGo7mcye+UWjjXLr7i0cDEszAloQ9aQ== srangana@redhat.com", "key_options": null, "keyfile": "/home/shyamsundarr/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "shyamsundarr", "validate_certs": true} 2021-11-11T20:39:16.219 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'shuzhenyi', 'ovpn': 'bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "shuzhenyi", "ovpn": "bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKRZyOnj6D9tCw5o4+ujpos5Uj1O0XVvea798DsXPRh8bEE1pLkARQJLv4iW6LsG3x/CXruahmwT+6C/VHppIsbgtpadMWdCTPYrsjctPK0/N9nr4kQEL6XeK3/GHFnwqYcc+LySpNNfbKrgN4qWOwdZBhQjSV7K7FJd4ZYSm4wp6TVbvKKp7o+JdeM2zhstfC9AcCtBv/TpxixFFq/+y8/Z0PwqN1YyHduDOr1dr1DWn4ZzrkhiTYfPOltx0jweD6zTJoTAJDqae7kBA+3XH3dUGX3n7Tchsa3bnIKVz9sLQvH/vPro61I+Xig6MQh77JsylNvpwb1LLOEzDf7dxl bigwhite@wrjj37deMacBook-Pro-2.local", "key_options": null, "keyfile": "/home/shuzhenyi/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "shuzhenyi", "validate_certs": true} 2021-11-11T20:39:16.221 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'shuzhenyi', 'ovpn': 'bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "shuzhenyi", "ovpn": "bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKRZyOnj6D9tCw5o4+ujpos5Uj1O0XVvea798DsXPRh8bEE1pLkARQJLv4iW6LsG3x/CXruahmwT+6C/VHppIsbgtpadMWdCTPYrsjctPK0/N9nr4kQEL6XeK3/GHFnwqYcc+LySpNNfbKrgN4qWOwdZBhQjSV7K7FJd4ZYSm4wp6TVbvKKp7o+JdeM2zhstfC9AcCtBv/TpxixFFq/+y8/Z0PwqN1YyHduDOr1dr1DWn4ZzrkhiTYfPOltx0jweD6zTJoTAJDqae7kBA+3XH3dUGX3n7Tchsa3bnIKVz9sLQvH/vPro61I+Xig6MQh77JsylNvpwb1LLOEzDf7dxl bigwhite@wrjj37deMacBook-Pro-2.local", "key_options": null, "keyfile": "/home/shuzhenyi/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "shuzhenyi", "validate_certs": true} 2021-11-11T20:39:16.447 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'mdipalma', 'ovpn': 'mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mdipalma", "ovpn": "mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDEcI3X9mNxtHRSvi2gVFavu21kRCWDd1erXW3d/1zWnoUvRxe9ft2xA1c3Bmo067GqhFudPcXoYJvz3atAGR9k3n1UTtTJe95XRZ1Zmn/boi6mn7I2oRVFpW/KjxaiM3iN2SHKknMtcs6qRTioapYCU7u924oc0izqPBvkrC73cfVF4emH1ERWEcQBTxWub6N6Mn3xaDmJ5xnGlNvDY70CC9osoa2VRNJM7GU9eUQClHirPbSooDvvVKfTGD7YfMH76j6PdDR/a/cgNeiMevtMbeFGhu1SOEHb1ihCNoGq2vQFPPWwwTCzMI05XuUEc4iaxw/kbGBMpWbmIvswIMQrRwgwLZk67GGpf/dEeutI6WseeNKNuHnTzzx43AW9G2coGq0RKE7OuA5Cqj0CO7Kcf5UA4AqIIroLETVBrrEV6qzpkD74QBZXPl7gbSZIyxLpvtQoMfSSzN+Mj184LmQoURCqIT+yM5gddF5/S5I+rpw81vBj5M9ZskyzVQAo0H8= michelledipalma@mdipalma-mac", "key_options": null, "keyfile": "/home/mdipalma/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mdipalma", "validate_certs": true} 2021-11-11T20:39:16.458 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'mdipalma', 'ovpn': 'mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mdipalma", "ovpn": "mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDEcI3X9mNxtHRSvi2gVFavu21kRCWDd1erXW3d/1zWnoUvRxe9ft2xA1c3Bmo067GqhFudPcXoYJvz3atAGR9k3n1UTtTJe95XRZ1Zmn/boi6mn7I2oRVFpW/KjxaiM3iN2SHKknMtcs6qRTioapYCU7u924oc0izqPBvkrC73cfVF4emH1ERWEcQBTxWub6N6Mn3xaDmJ5xnGlNvDY70CC9osoa2VRNJM7GU9eUQClHirPbSooDvvVKfTGD7YfMH76j6PdDR/a/cgNeiMevtMbeFGhu1SOEHb1ihCNoGq2vQFPPWwwTCzMI05XuUEc4iaxw/kbGBMpWbmIvswIMQrRwgwLZk67GGpf/dEeutI6WseeNKNuHnTzzx43AW9G2coGq0RKE7OuA5Cqj0CO7Kcf5UA4AqIIroLETVBrrEV6qzpkD74QBZXPl7gbSZIyxLpvtQoMfSSzN+Mj184LmQoURCqIT+yM5gddF5/S5I+rpw81vBj5M9ZskyzVQAo0H8= michelledipalma@mdipalma-mac", "key_options": null, "keyfile": "/home/mdipalma/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mdipalma", "validate_certs": true} 2021-11-11T20:39:16.676 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'kchang', 'ovpn': 'kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kchang", "ovpn": "kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+tqgRGdlbb7NMefZVPCtbzVZBapxmmPcKaUtqT7E/9AKRoHuroZQ7KJQfO15POpIfEja7BugBt5F95aW2Bd9/xhulzC4/JjiZYc6ST5Na03BrcudI3YcktoGR8Mm7ZVzSojvm0SKtLLHM5Su17JpCC7BitMdao1mETlUOklEXc6pgbr9XHOayo1zeuos8DB90iA8TQ2soR/POD1Rix4GKniCHXz10+iXQFYPsif1HsQ3rh8Vi1M0cTq98bMSmfsKODil8KIxf+lvLCL4GAgDYEyxnstiTFh5GAcIMOOo2dl1fDb/Oa981CxWU+xxJD+M7AEJVeyAqqAnShvZU6y1L kiefer@kiefer-uchi", "key_options": null, "keyfile": "/home/kchang/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kchang", "validate_certs": true} 2021-11-11T20:39:16.687 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'kchang', 'ovpn': 'kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kchang", "ovpn": "kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+tqgRGdlbb7NMefZVPCtbzVZBapxmmPcKaUtqT7E/9AKRoHuroZQ7KJQfO15POpIfEja7BugBt5F95aW2Bd9/xhulzC4/JjiZYc6ST5Na03BrcudI3YcktoGR8Mm7ZVzSojvm0SKtLLHM5Su17JpCC7BitMdao1mETlUOklEXc6pgbr9XHOayo1zeuos8DB90iA8TQ2soR/POD1Rix4GKniCHXz10+iXQFYPsif1HsQ3rh8Vi1M0cTq98bMSmfsKODil8KIxf+lvLCL4GAgDYEyxnstiTFh5GAcIMOOo2dl1fDb/Oa981CxWU+xxJD+M7AEJVeyAqqAnShvZU6y1L kiefer@kiefer-uchi", "key_options": null, "keyfile": "/home/kchang/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kchang", "validate_certs": true} 2021-11-11T20:39:16.915 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'vereddy', 'ovpn': 'vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vereddy", "ovpn": "vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDjj3bbbLvd4IHeFrSJyM9N6uKXA7ElQrzfcisKRGjUPON8IsAWkXQeAnHoBsIQv7tRGudXOrK+tbfv7crLI0/CxL+o0cCVW8bxuTEtdz07ZFsND9cppCc7Yq+wHsg7nQ/n0QfxnlYc0T9pATXUoT+aW4c3NQoDbsupLSkNs1C+G/MEBzz6iH6tgrv43OAOUccfQbC+IYx4RIKL5MteEjVrZanNELIG+jsYe+CZLdsONk1nR8+xKQw19Sm/jkKdl1K9NXoVPEBQmFXrYl6G+7tCdWeU3Vq24lM1X5p1mVspcG6FyyctcuP/8+zhL2L208DrXlOVPcqHR/4FEf3Wmk89 vereddy@vereddy.remote.csb", "key_options": null, "keyfile": "/home/vereddy/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vereddy", "validate_certs": true} 2021-11-11T20:39:16.917 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'vereddy', 'ovpn': 'vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "vereddy", "ovpn": "vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDjj3bbbLvd4IHeFrSJyM9N6uKXA7ElQrzfcisKRGjUPON8IsAWkXQeAnHoBsIQv7tRGudXOrK+tbfv7crLI0/CxL+o0cCVW8bxuTEtdz07ZFsND9cppCc7Yq+wHsg7nQ/n0QfxnlYc0T9pATXUoT+aW4c3NQoDbsupLSkNs1C+G/MEBzz6iH6tgrv43OAOUccfQbC+IYx4RIKL5MteEjVrZanNELIG+jsYe+CZLdsONk1nR8+xKQw19Sm/jkKdl1K9NXoVPEBQmFXrYl6G+7tCdWeU3Vq24lM1X5p1mVspcG6FyyctcuP/8+zhL2L208DrXlOVPcqHR/4FEf3Wmk89 vereddy@vereddy.remote.csb", "key_options": null, "keyfile": "/home/vereddy/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "vereddy", "validate_certs": true} 2021-11-11T20:39:17.143 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'benhanokh', 'ovpn': 'gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "benhanokh", "ovpn": "gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+EkUbnaeC7Dqtq2N4/dX9v+WGlrgRS2ygHhlbZy4KHbd6+Q9T5NAWtqnFwqcbFpYYUBwMvz8zLo6HYN6gNskRN0qTiFKPQdtnydF0Tx1KHrdGGcX3G2lxyGH5P8nrqReXAPPvRiVc2/Mj7APCTmY2JftkSS1qYcTleeueivVyho6z6uD1AyC7OXANhtn7kGeIY/NvVKBhDr+3TuPmpPwPCQMcJUyZWCHYyp47v2vzNBy2Etp7WjGwOnfZ5ULiJGjH6PCXx0YfpmTUOPVkNbV6Om+0YJ3fcLlmQ/2nsdsWfOKvoAr+1uzO81h7ZBX9LVEitj8YU3WeSCV2vqS2VvOpq5NyHXWGq+ZkOCfhy4Z9MvMG+6hqp4p4z1JMpKsZ191kP7eAQua41GkTSQPlEfF+um1UV6ZjNzhB68utOXyGDoDcxct9oOdnhhjwu5W4/0t9iBHuMn02iHRYVrUWOH2FXVysR0p5x/L2xdvNPiTgFAZ2eN+EFWmdan4e9mAAw3M= gbenhano@localhost.localdomain", "key_options": null, "keyfile": "/home/benhanokh/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "benhanokh", "validate_certs": true} 2021-11-11T20:39:17.145 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'benhanokh', 'ovpn': 'gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "benhanokh", "ovpn": "gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+EkUbnaeC7Dqtq2N4/dX9v+WGlrgRS2ygHhlbZy4KHbd6+Q9T5NAWtqnFwqcbFpYYUBwMvz8zLo6HYN6gNskRN0qTiFKPQdtnydF0Tx1KHrdGGcX3G2lxyGH5P8nrqReXAPPvRiVc2/Mj7APCTmY2JftkSS1qYcTleeueivVyho6z6uD1AyC7OXANhtn7kGeIY/NvVKBhDr+3TuPmpPwPCQMcJUyZWCHYyp47v2vzNBy2Etp7WjGwOnfZ5ULiJGjH6PCXx0YfpmTUOPVkNbV6Om+0YJ3fcLlmQ/2nsdsWfOKvoAr+1uzO81h7ZBX9LVEitj8YU3WeSCV2vqS2VvOpq5NyHXWGq+ZkOCfhy4Z9MvMG+6hqp4p4z1JMpKsZ191kP7eAQua41GkTSQPlEfF+um1UV6ZjNzhB68utOXyGDoDcxct9oOdnhhjwu5W4/0t9iBHuMn02iHRYVrUWOH2FXVysR0p5x/L2xdvNPiTgFAZ2eN+EFWmdan4e9mAAw3M= gbenhano@localhost.localdomain", "key_options": null, "keyfile": "/home/benhanokh/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "benhanokh", "validate_certs": true} 2021-11-11T20:39:17.372 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jsalomon', 'ovpn': 'jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jsalomon", "ovpn": "jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBYtdHKBXQcXQFkEW8fSwlFK/xZM16VcFY288ABI/DPfejTStHhxiRvmlsWzMy0Ar5WadZ8FU1ptjA3MKBEY1Do1r1Ul369k3zHCvxXyg9QGGMNPYfHmnNeoF4GEgoJpuounPs0mkLmrPTR+cR+/XuaZZt38nsad8M/zJetnolvqCWpcV87D7+Rt8F32J6kGDknC4ikEe8I0n7F+goOmJhlgkqegbKEeJ/EMnOItwVZHFwaC+sxOq4IlYp2f1aZMKGAtr5+ICwYH1SMCcvnp3qWL8O//t2tvguZgQxzIs6Nf5GaxwdQkh4CvjbWiCkIbGFJ57YICS6k++KMXMbEnwh jsalomon@Josh-laptop.tlv.redhat.com", "key_options": null, "keyfile": "/home/jsalomon/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jsalomon", "validate_certs": true} 2021-11-11T20:39:17.383 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jsalomon', 'ovpn': 'jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jsalomon", "ovpn": "jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBYtdHKBXQcXQFkEW8fSwlFK/xZM16VcFY288ABI/DPfejTStHhxiRvmlsWzMy0Ar5WadZ8FU1ptjA3MKBEY1Do1r1Ul369k3zHCvxXyg9QGGMNPYfHmnNeoF4GEgoJpuounPs0mkLmrPTR+cR+/XuaZZt38nsad8M/zJetnolvqCWpcV87D7+Rt8F32J6kGDknC4ikEe8I0n7F+goOmJhlgkqegbKEeJ/EMnOItwVZHFwaC+sxOq4IlYp2f1aZMKGAtr5+ICwYH1SMCcvnp3qWL8O//t2tvguZgQxzIs6Nf5GaxwdQkh4CvjbWiCkIbGFJ57YICS6k++KMXMbEnwh jsalomon@Josh-laptop.tlv.redhat.com", "key_options": null, "keyfile": "/home/jsalomon/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jsalomon", "validate_certs": true} 2021-11-11T20:39:17.603 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'jolmomar', 'ovpn': 'jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jolmomar", "ovpn": "jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD9ltUNitnn0FNwMeiGkBnozpBN5ziVR2MreaqkfqTQY2TgI24559xuILyy+TnAKumhemZ8i8nIQ876Iso85OzIXbUbRM45hpL/bMQX7pWyMIRNeFKrCDwf18OT0DPc8aeDOAFQ6YoN0j4awUENjdZuuAW80cfcjoQO3m7ORF8wBF0IvMWwPDIQQrz+Li31ECgXg6VC7K3e+BNHlfNQ/2/ahS0+V4J6JYFsylZL0r2tydhDMd6rR8QA9c9BgeBz7+KWobGtiZVe7KReTzJ0O38PxNd8br3aS50j5BiN63gch8eFxPAROaUMnuiavu6OTDy4/aeL3Zc5WpUy3VyyumWf jolmomar@localhost.localdomain", "key_options": null, "keyfile": "/home/jolmomar/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jolmomar", "validate_certs": true} 2021-11-11T20:39:17.604 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'jolmomar', 'ovpn': 'jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jolmomar", "ovpn": "jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD9ltUNitnn0FNwMeiGkBnozpBN5ziVR2MreaqkfqTQY2TgI24559xuILyy+TnAKumhemZ8i8nIQ876Iso85OzIXbUbRM45hpL/bMQX7pWyMIRNeFKrCDwf18OT0DPc8aeDOAFQ6YoN0j4awUENjdZuuAW80cfcjoQO3m7ORF8wBF0IvMWwPDIQQrz+Li31ECgXg6VC7K3e+BNHlfNQ/2/ahS0+V4J6JYFsylZL0r2tydhDMd6rR8QA9c9BgeBz7+KWobGtiZVe7KReTzJ0O38PxNd8br3aS50j5BiN63gch8eFxPAROaUMnuiavu6OTDy4/aeL3Zc5WpUy3VyyumWf jolmomar@localhost.localdomain", "key_options": null, "keyfile": "/home/jolmomar/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jolmomar", "validate_certs": true} 2021-11-11T20:39:17.831 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'julpark', 'ovpn': 'julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "julpark", "ovpn": "julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDoefxOfeArToGucfWG7y1tyLF2I1J/IkBSlmxxEJyHyDUB6ElXjYIJReoiPfmB0xLEmINpmt8QA4XN/8UbR1r9LFOexlZZbX1LSbihi11LXCvfIJWqkHVwIyKHRTu/duMrntBFVENJ0LVf0GSWVTtllXaDA+fMkp6/ra/hlRHQbx19UXy/9+Pogj8Xl+SntnMGiHlX8FFq2A2tUqhdQSf/1GW1HfUIa5WtwdEqIpKpAgMM7sOW9i3QAmLAMo8HiKrArSiy7jrRUODCqsTJsfifxEbmnOS0ZtmDh0eeWL/Dyqbz99Rg4WbFNeK8oyZYr7D6bEAhd2az2A+wycPx4Nq7 julpark@julpark.remote.csb", "key_options": null, "keyfile": "/home/julpark/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "julpark", "validate_certs": true} 2021-11-11T20:39:17.842 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'julpark', 'ovpn': 'julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "julpark", "ovpn": "julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDoefxOfeArToGucfWG7y1tyLF2I1J/IkBSlmxxEJyHyDUB6ElXjYIJReoiPfmB0xLEmINpmt8QA4XN/8UbR1r9LFOexlZZbX1LSbihi11LXCvfIJWqkHVwIyKHRTu/duMrntBFVENJ0LVf0GSWVTtllXaDA+fMkp6/ra/hlRHQbx19UXy/9+Pogj8Xl+SntnMGiHlX8FFq2A2tUqhdQSf/1GW1HfUIa5WtwdEqIpKpAgMM7sOW9i3QAmLAMo8HiKrArSiy7jrRUODCqsTJsfifxEbmnOS0ZtmDh0eeWL/Dyqbz99Rg4WbFNeK8oyZYr7D6bEAhd2az2A+wycPx4Nq7 julpark@julpark.remote.csb", "key_options": null, "keyfile": "/home/julpark/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "julpark", "validate_certs": true} 2021-11-11T20:39:18.082 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'pdhange', 'ovpn': 'pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pdhange", "ovpn": "pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD03otezqmbR7v9dq62mWH5Kkg692jdmDZk6Ve4vux0A0YhxBuXDCyBWLE1Z4lLNxQahvK0pGsgokp+Uk+Q6aU+ZnF/E2jLp6YWerFXIHY3M7byObJx9+QuVBfNI0bZBkV3+vSjhSCGOYU+TrTHsKZqhsTcejZjmL3gFLmWfU6dTh0sX05qz97cdJl3CMPhCIR3KjtkiPygj417/N7wet/Efv1t79TvyS7dtK1oz+zRLEpKi/fZjvpXb8xec7as72Y0XexsWMDyZJ/N8PijTQ5L6EfLMV7ae/bWGzNAKsoC1dHVY7q/Q4pzktC6zU0Rc9sDRMtrgFg+q+04BfveSLfxTb9tqf7SbSvh7IAG1eKiEuDT/uy3T6Uiw+CiZ0eIqYzZscO/aDjUArGbS8eyaycybdYCBK9Jz38i0mYKkXg855yoVuIExr6MXieYwyVoVW8gbx1bAslaafrzra2dkAFByOJ0sbosPFRgR2+boUSxD6IaE+ALgno4T/0CI2RMiH0= prashant@cephdev\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4m0LAcC7XiFiXFtYcCvZqX1CtyV9JU+VUEawZJpWrGTOr8c+vIexq7+WKg+eIUMot4+dtfwG8lByS1VofglIe5/quv17Xj7dO/hF2WkV8O7RYt7NhzmrKg54yB73pbRpvqzGceWWDvGRsWQGzbY7ARqOp4FPIDJvgXABCkak+hmjXyXpvXN418CBDbDmxHWwFb9QtB/MpQFiFrfl1EMDj8DaXZNWgo5J/qytic6LgiyPHhHfhSylk4cl9TGftNatobpMjqBT0O2/plk/qf+7pHPuvGSgihjijdpEPzVfctXE2OmsQAIZgkzXeCqts+s6ZYdUKq1FvpV5rMM8ZLqfCkDjEzFENJS68YYE2EjxAEgMzj7h/sWc4B5I5pxkAqVAQSGjYRu6rCCbFz8PXUdTyW+Vs1YeylnlBMiCa4xE4gwvW3HlWEnumtyURd1PRENrGJsJ/0uX6bpltbSg4sGACRYpLPJMm3OhqI3Jm/J/e2NLjfCw+svU2447xnfJOh68= pdhange@vaquita\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCoeyVaMjhcGrWQB+4Rq+SSIi8+OY/kx2s1vofiGt1gu3RNtFmG0Df1VITJJ32dbzaW7kTNxaRvhldNd/n7yEMssG+Wg+a5WKvSudlbVMmscbSXdFuVbcCW1jNrTGCYrxL7Sn5e7NQEQfAsCbfXwTcYBw33NdBVovC+gs2V3sZGvX3Y0akZONNuVg1mhu0L4m1/5IegJh005KeilwetHA2A5Dr10RG6oyIgT7t4z5gZXsrOiG3UQBXaekILYV9jxMWoFXGkVLc8TpL9SUAdxE3Owc0u1XiPqYYlEN+zDvE0554Zj8Oc2ZO6HRXCZXQFvzXeCD12YswcOfkIDmJRF7GT prashant@mauli", "key_options": null, "k 2021-11-11T20:39:18.084 INFO:teuthology.task.ansible.out:eyfile": "/home/pdhange/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pdhange", "validate_certs": true} 2021-11-11T20:39:18.113 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'pdhange', 'ovpn': 'pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pdhange", "ovpn": "pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD03otezqmbR7v9dq62mWH5Kkg692jdmDZk6Ve4vux0A0YhxBuXDCyBWLE1Z4lLNxQahvK0pGsgokp+Uk+Q6aU+ZnF/E2jLp6YWerFXIHY3M7byObJx9+QuVBfNI0bZBkV3+vSjhSCGOYU+TrTHsKZqhsTcejZjmL3gFLmWfU6dTh0sX05qz97cdJl3CMPhCIR3KjtkiPygj417/N7wet/Efv1t79TvyS7dtK1oz+zRLEpKi/fZjvpXb8xec7as72Y0XexsWMDyZJ/N8PijTQ5L6EfLMV7ae/bWGzNAKsoC1dHVY7q/Q4pzktC6zU0Rc9sDRMtrgFg+q+04BfveSLfxTb9tqf7SbSvh7IAG1eKiEuDT/uy3T6Uiw+CiZ0eIqYzZscO/aDjUArGbS8eyaycybdYCBK9Jz38i0mYKkXg855yoVuIExr6MXieYwyVoVW8gbx1bAslaafrzra2dkAFByOJ0sbosPFRgR2+boUSxD6IaE+ALgno4T/0CI2RMiH0= prashant@cephdev\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4m0LAcC7XiFiXFtYcCvZqX1CtyV9JU+VUEawZJpWrGTOr8c+vIexq7+WKg+eIUMot4+dtfwG8lByS1VofglIe5/quv17Xj7dO/hF2WkV8O7RYt7NhzmrKg54yB73pbRpvqzGceWWDvGRsWQGzbY7ARqOp4FPIDJvgXABCkak+hmjXyXpvXN418CBDbDmxHWwFb9QtB/MpQFiFrfl1EMDj8DaXZNWgo5J/qytic6LgiyPHhHfhSylk4cl9TGftNatobpMjqBT0O2/plk/qf+7pHPuvGSgihjijdpEPzVfctXE2OmsQAIZgkzXeCqts+s6ZYdUKq1FvpV5rMM8ZLqfCkDjEzFENJS68YYE2EjxAEgMzj7h/sWc4B5I5pxkAqVAQSGjYRu6rCCbFz8PXUdTyW+Vs1YeylnlBMiCa4xE4gwvW3HlWEnumtyURd1PRENrGJsJ/0uX6bpltbSg4sGACRYpLPJMm3OhqI3Jm/J/e2NLjfCw+svU2447xnfJOh68= pdhange@vaquita\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCoeyVaMjhcGrWQB+4Rq+SSIi8+OY/kx2s1vofiGt1gu3RNtFmG0Df1VITJJ32dbzaW7kTNxaRvhldNd/n7yEMssG+Wg+a5WKvSudlbVMmscbSXdFuVbcCW1jNrTGCYrxL7Sn5e7NQEQfAsCbfXwTcYBw33NdBVovC+gs2V3sZGvX3Y0akZONNuVg1mhu0L4m1/5IegJh005KeilwetHA2A5Dr10RG6oyIgT7t4z5gZXsrOiG3UQBXaekILYV9jxMWoFXGkVLc8TpL9SUAdxE3Owc0u1XiPqYYlEN+zDvE0554Zj8Oc2ZO6HRXCZXQFvzXeCD12YswcOfkIDmJRF7GT prashant@mauli", "key_options": null, "k 2021-11-11T20:39:18.115 INFO:teuthology.task.ansible.out:eyfile": "/home/pdhange/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pdhange", "validate_certs": true} 2021-11-11T20:39:18.324 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dpivonka', 'ovpn': 'dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dpivonka", "ovpn": "dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg7xW5+Mk7pG5MjLxOc7GTvDTCPBJj2L3MEQZIapmmo6t8uPODwDewXncSzrKDH6zFVE4TTTD9fJdu+BhyT+jCFgXrEOMMabj7vJsg0wO0SWqr+4NECFPhD9y4ewfYiewBeXisclA5Bto91v93AtYpZBCdmPYSi0BEcPVc6S3iBUD0PzIrL62I/kBAtm0f/bScRiT7XvQJXvkbEObW2yS2XhUv2qgq1VlFvO1jl3B85KMTHVb8+LrKpQWOKfRfxJ74c51y6tnEjZLFO6Xn0JevqOhd8gPQIXueFZOM0g0pNp1mbtGaaBlEIibbrzo2p9kzsdvJeHMq1XuMIi1/Qyj9 dpivonka@dhcp-41-144.bos.redhat.com", "key_options": null, "keyfile": "/home/dpivonka/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dpivonka", "validate_certs": true} 2021-11-11T20:39:18.345 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dpivonka', 'ovpn': 'dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dpivonka", "ovpn": "dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg7xW5+Mk7pG5MjLxOc7GTvDTCPBJj2L3MEQZIapmmo6t8uPODwDewXncSzrKDH6zFVE4TTTD9fJdu+BhyT+jCFgXrEOMMabj7vJsg0wO0SWqr+4NECFPhD9y4ewfYiewBeXisclA5Bto91v93AtYpZBCdmPYSi0BEcPVc6S3iBUD0PzIrL62I/kBAtm0f/bScRiT7XvQJXvkbEObW2yS2XhUv2qgq1VlFvO1jl3B85KMTHVb8+LrKpQWOKfRfxJ74c51y6tnEjZLFO6Xn0JevqOhd8gPQIXueFZOM0g0pNp1mbtGaaBlEIibbrzo2p9kzsdvJeHMq1XuMIi1/Qyj9 dpivonka@dhcp-41-144.bos.redhat.com", "key_options": null, "keyfile": "/home/dpivonka/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dpivonka", "validate_certs": true} 2021-11-11T20:39:18.543 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'adking', 'ovpn': 'adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "adking", "ovpn": "adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0m+8wna4RnrOcuCUj971EBe4cFiTXVkEWkIsUX3yeCUwMaS/ZjaoOjcdYX7cYpCp1ilo4FSO5BsochU0RiMbQEwLGcE0zB2q1ac8+yYKCuWWyknig2t4bOReu5OrXvl6mj/9R+p7clFlYhwIoNG5ZZ/2eTbo3nH7gWgaOKaPbTtvEGg/5R/qjQKg8OTC8oix6kLsaNTisQJ78Zd7b4ukg9bO0jlq/2UhQOBM072sU1/kJpWTkcA0xxm3nsC4rWPwapgz7UwBmMdyx116GZgWdfQecTLKO7b8pyJw21npk2ntUmeMi7WPZGX6VI3IpaAhBjW3KiATAb8JbAQh8F9bmejkw6x7zrP3hYKQOg3A+U8DKRRo/Ad+f0zz9gjC42DUvMEOr09RBdxZcEAPY7H/YRWEqbA2zPX3Gfth4b7/letzM3m9ToUvVfSt1CdjaZD+gIia9pCgDMb+j1j2w9reG4F7Us+W/1f5pYCyFyrm4XO3AdoYM9/FOAOR3vffTIQU= adking@localhost.localdomain\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJOefD/VJfS15LRyuVDLluKv93Zgocsl8BhiDMimvBNbrGWc795v2+eevWuwaL/SlL44SnsNZ9Mz8FVn9YuFoQiHLg7cHZNLg9EJh5SMXzZzwLZHnF02MdI2CU6U++9nmh8FtBrdBWKzZ7o+01wOw+52dYOxvP7IJ7C/vjwOLgfigEN8Nd1PrxALVNIQz7fsiet0n3odk9PJfoqTu3IloXm9qbQiPFyfMA6b8a20RdLZCJDGGG5rd6gtsyeFHoAPZEz8FEUlNkow7NM7ZhdxOz/fWXyMZ27KmY8WSAgflbxysMzexyifkGzypiE3NQ10hdPGBgoYsHrAN4Qmte1/Z4aI5nxrNzLy/uNc+L/Bp/0XkLj9zfoiFFOReXvlz/laAV6LpbHyacr+iCwv9rlI7echXmvC6mSJXfBPuyR1uromUXpYRhEkn5uiOsCIQCuYpx2jeil7JV+LJlF62lLSndOv4g12Gq2TEZ4Bhsr2902kftPkLFenyfDTh3A2qGGNM= adking@localhost.localdomain", "key_options": null, "keyfile": "/home/adking/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "adking", "validate_certs": true} 2021-11-11T20:39:18.585 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'adking', 'ovpn': 'adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "adking", "ovpn": "adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0m+8wna4RnrOcuCUj971EBe4cFiTXVkEWkIsUX3yeCUwMaS/ZjaoOjcdYX7cYpCp1ilo4FSO5BsochU0RiMbQEwLGcE0zB2q1ac8+yYKCuWWyknig2t4bOReu5OrXvl6mj/9R+p7clFlYhwIoNG5ZZ/2eTbo3nH7gWgaOKaPbTtvEGg/5R/qjQKg8OTC8oix6kLsaNTisQJ78Zd7b4ukg9bO0jlq/2UhQOBM072sU1/kJpWTkcA0xxm3nsC4rWPwapgz7UwBmMdyx116GZgWdfQecTLKO7b8pyJw21npk2ntUmeMi7WPZGX6VI3IpaAhBjW3KiATAb8JbAQh8F9bmejkw6x7zrP3hYKQOg3A+U8DKRRo/Ad+f0zz9gjC42DUvMEOr09RBdxZcEAPY7H/YRWEqbA2zPX3Gfth4b7/letzM3m9ToUvVfSt1CdjaZD+gIia9pCgDMb+j1j2w9reG4F7Us+W/1f5pYCyFyrm4XO3AdoYM9/FOAOR3vffTIQU= adking@localhost.localdomain\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJOefD/VJfS15LRyuVDLluKv93Zgocsl8BhiDMimvBNbrGWc795v2+eevWuwaL/SlL44SnsNZ9Mz8FVn9YuFoQiHLg7cHZNLg9EJh5SMXzZzwLZHnF02MdI2CU6U++9nmh8FtBrdBWKzZ7o+01wOw+52dYOxvP7IJ7C/vjwOLgfigEN8Nd1PrxALVNIQz7fsiet0n3odk9PJfoqTu3IloXm9qbQiPFyfMA6b8a20RdLZCJDGGG5rd6gtsyeFHoAPZEz8FEUlNkow7NM7ZhdxOz/fWXyMZ27KmY8WSAgflbxysMzexyifkGzypiE3NQ10hdPGBgoYsHrAN4Qmte1/Z4aI5nxrNzLy/uNc+L/Bp/0XkLj9zfoiFFOReXvlz/laAV6LpbHyacr+iCwv9rlI7echXmvC6mSJXfBPuyR1uromUXpYRhEkn5uiOsCIQCuYpx2jeil7JV+LJlF62lLSndOv4g12Gq2TEZ4Bhsr2902kftPkLFenyfDTh3A2qGGNM= adking@localhost.localdomain", "key_options": null, "keyfile": "/home/adking/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "adking", "validate_certs": true} 2021-11-11T20:39:18.774 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'amathuri', 'ovpn': 'amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "amathuri", "ovpn": "amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCs6q2b5NcUsNXl+w1Mrc7idNvGdwWkL0XRX5zTw22UNpQyqPkSrRTFKi4NxoY8ucv8JoLnaJL7hYSRHKTh7SqH78uPLZX0FLtoHvpYLgUgJzzJ5AOW5uekIhCMCm+3egmFnEBwFcNxtGnU2es02ByXNEXRxt4/LQ0f8K6lMqxVRSKYcltvWzlA7WFFdTnYMunbCYOdpw88RXm2pYjGIHRBnie7+pFiFSS6K/mkzQcfLiJmahgLSaYslI0ho1vQd8xfNLmZ700C5F4VTUlT/mJK/dI2LF3OfaPTsyqKNzOchrjLFx5b6dhZ4WZyqu7VKsHNm+xHZwT8VP3LPGSVVBdmEsiCsmV4g4QTI5ssZLi7siZDpaFp72nVAIO1i6kBGuiF3or+J4LmIh8ZRKzTc0DRDOw7S+IkWu1fv2UhYg83z6wrz4HeczI+bnolFFWW6HgGV3MFaUeRTRvrbxw2DPoEGQ+cjP0wsjyEhxlScwdh8W8IbVpNl+7mBe45YGUH2hg83Lst5QQxD4PHelriu40moWRHFKCnKfNesb4L1uqu29o8f+/PgpLdrWE/P5zIKe2ReWJyScYAI7jTMlFw+xgf+0TKryk44NMy90l3ei0umlx4qsFbbEv+lhzAALrUJ/da7fsQw7eIVykmcscBeYr99/YSYk4MGdk4jVCiHsQkdw== amathuri@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJ3ch+pU88/tX5TVVFl/1pPief8drYLRkK4BtfJk+xVNxX9qkDtOtiLAU+OugYF1/xk9hE9zTbm3UBL23unrzhlR80t8f2FiF1YFhHVzeZ/p/OunmkU0RFQzkMVuwKVOUNulw9FwpEZ/0Bjfxvaims0SBWaCzBgE5g0R+Ofdq/bVF82K8vU5Laa3KZ/cCdGzFgW00cBvPAKIpQ93g8TRqsCsFvR37nY9v5SU1hJ9araEp6LRA86tUt3EFtxD2tRcmqKGtCb0kOr+rRXWVGiaUBiedHxFudIy0+TNj4fNrJwpoC0hcBO9ek4Bh6hM7Zqzum9HpkPGlz84jkm+r+xMThaZCh8iFkgxyK+bw8B8smaVJD04ob9Wb4RYoFYfQ7TEkPkO9gr+UsSesbP9fWfucemUqwK0htOOtpc1YRwdOPcdPJWAfx9J5+YVL8BTWYBaTdmHslOKeUGM5fCob6QxC6Er9xweys9jh7MXttSpe013bY7P4ZDRF8uWvygpsytPE= amathuri@redhat.com", "key_ 2021-11-11T20:39:18.776 INFO:teuthology.task.ansible.out:options": null, "keyfile": "/home/amathuri/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "amathuri", "validate_certs": true} 2021-11-11T20:39:18.816 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'amathuri', 'ovpn': 'amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "amathuri", "ovpn": "amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCs6q2b5NcUsNXl+w1Mrc7idNvGdwWkL0XRX5zTw22UNpQyqPkSrRTFKi4NxoY8ucv8JoLnaJL7hYSRHKTh7SqH78uPLZX0FLtoHvpYLgUgJzzJ5AOW5uekIhCMCm+3egmFnEBwFcNxtGnU2es02ByXNEXRxt4/LQ0f8K6lMqxVRSKYcltvWzlA7WFFdTnYMunbCYOdpw88RXm2pYjGIHRBnie7+pFiFSS6K/mkzQcfLiJmahgLSaYslI0ho1vQd8xfNLmZ700C5F4VTUlT/mJK/dI2LF3OfaPTsyqKNzOchrjLFx5b6dhZ4WZyqu7VKsHNm+xHZwT8VP3LPGSVVBdmEsiCsmV4g4QTI5ssZLi7siZDpaFp72nVAIO1i6kBGuiF3or+J4LmIh8ZRKzTc0DRDOw7S+IkWu1fv2UhYg83z6wrz4HeczI+bnolFFWW6HgGV3MFaUeRTRvrbxw2DPoEGQ+cjP0wsjyEhxlScwdh8W8IbVpNl+7mBe45YGUH2hg83Lst5QQxD4PHelriu40moWRHFKCnKfNesb4L1uqu29o8f+/PgpLdrWE/P5zIKe2ReWJyScYAI7jTMlFw+xgf+0TKryk44NMy90l3ei0umlx4qsFbbEv+lhzAALrUJ/da7fsQw7eIVykmcscBeYr99/YSYk4MGdk4jVCiHsQkdw== amathuri@redhat.com\nssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJ3ch+pU88/tX5TVVFl/1pPief8drYLRkK4BtfJk+xVNxX9qkDtOtiLAU+OugYF1/xk9hE9zTbm3UBL23unrzhlR80t8f2FiF1YFhHVzeZ/p/OunmkU0RFQzkMVuwKVOUNulw9FwpEZ/0Bjfxvaims0SBWaCzBgE5g0R+Ofdq/bVF82K8vU5Laa3KZ/cCdGzFgW00cBvPAKIpQ93g8TRqsCsFvR37nY9v5SU1hJ9araEp6LRA86tUt3EFtxD2tRcmqKGtCb0kOr+rRXWVGiaUBiedHxFudIy0+TNj4fNrJwpoC0hcBO9ek4Bh6hM7Zqzum9HpkPGlz84jkm+r+xMThaZCh8iFkgxyK+bw8B8smaVJD04ob9Wb4RYoFYfQ7TEkPkO9gr+UsSesbP9fWfucemUqwK0htOOtpc1YRwdOPcdPJWAfx9J5+YVL8BTWYBaTdmHslOKeUGM5fCob6QxC6Er9xweys9jh7MXttSpe013bY7P4ZDRF8uWvygpsytPE= amathuri@redhat.com", "key_ 2021-11-11T20:39:18.818 INFO:teuthology.task.ansible.out:options": null, "keyfile": "/home/amathuri/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "amathuri", "validate_certs": true} 2021-11-11T20:39:19.014 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'nizamial09', 'ovpn': 'nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "nizamial09", "ovpn": "nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCt4sQTrDDg562IWY9Bw2fxFR7Y7b5lfo9egHa1qNFR5K4gt8lUwQXgzcxZK5dgSTZXHDGxrbYKWsdb8NbpgU/YjSqDUIptMZrhTI6pdHKZ2usY+nieaJj0NIZP/S7M93BbJeoNBYACjvDUr+vXEg6ZL3uAj+zFXr/4te+p4NqJBl4bSpvjcRK12O+F6HE3Q93j0Vlfs8IbXP4vcPcqOW+2usXw7JPdfjwuACJxrLTx/0mBsv8qXJDfOpPIHAMA/9SdTAzhs6XMo+fOwVOKkV+ki2fO3sl8pE16aFOkI3cYX36nkYYpZ2bJLnIPcLv37i3Nypl8eNLA4gzjkDf1o/KLO5vXOs76H189WGLqjh0SHAx/3KO6mrIhTq8Bv6YWoT3ntg22fXaxeajNhHcQ0bKZtGOAb0W1FcMRbSTkgEFAassYxlpTtuaCDIuA4fUVKW97Zu62eDDsSUtTWak/Yf940qAbptPPAw4ljmrvZRcjM9y6mGU8lKi5H4QoEOfbPok= nia@fedora", "key_options": null, "keyfile": "/home/nizamial09/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "nizamial09", "validate_certs": true} 2021-11-11T20:39:19.056 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'nizamial09', 'ovpn': 'nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "nizamial09", "ovpn": "nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCt4sQTrDDg562IWY9Bw2fxFR7Y7b5lfo9egHa1qNFR5K4gt8lUwQXgzcxZK5dgSTZXHDGxrbYKWsdb8NbpgU/YjSqDUIptMZrhTI6pdHKZ2usY+nieaJj0NIZP/S7M93BbJeoNBYACjvDUr+vXEg6ZL3uAj+zFXr/4te+p4NqJBl4bSpvjcRK12O+F6HE3Q93j0Vlfs8IbXP4vcPcqOW+2usXw7JPdfjwuACJxrLTx/0mBsv8qXJDfOpPIHAMA/9SdTAzhs6XMo+fOwVOKkV+ki2fO3sl8pE16aFOkI3cYX36nkYYpZ2bJLnIPcLv37i3Nypl8eNLA4gzjkDf1o/KLO5vXOs76H189WGLqjh0SHAx/3KO6mrIhTq8Bv6YWoT3ntg22fXaxeajNhHcQ0bKZtGOAb0W1FcMRbSTkgEFAassYxlpTtuaCDIuA4fUVKW97Zu62eDDsSUtTWak/Yf940qAbptPPAw4ljmrvZRcjM9y6mGU8lKi5H4QoEOfbPok= nia@fedora", "key_options": null, "keyfile": "/home/nizamial09/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "nizamial09", "validate_certs": true} 2021-11-11T20:39:19.254 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'dachary', 'ovpn': 'loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dachary", "ovpn": "loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC//ZCaEeVN/J09FtsvuZ4eEs4rTBtYoQyG+ZpuWkJLL90VyJeYrZzRsgvveKW2W9oFxCFcMy7IBhLes0TZ/X/4Ox9eUmP/eyqK8fBqT3K1F1W/uOmKLFhlQhkxxj+p9BOMBD5trfES/QJQ6UFSYIS2WhWaitDgONjoLMR6wXZ9r7yZeQBxbmG4wGI0Cwx4Wgg+I59UbE/5iUfnX++gN3rXrPnv9V0yFeciS4HvXRFXdrIzvQFHcZMmpOOmGruHXN/ME2dZCG+kaZhZdI6kICSlOwpU7TU1lv99zjBCeEzGfFkjv041ykKdNVR3qE0k5u0AOMqxJBORL/woSCdGvrBXqju2G/2ohl3B4zD6TAve+JLfWOV5C20NFRPPbTd53JaNpV9OQqwOfVniowzSYYBdedjm9sq6YwhJt8iBXze3OA8ZO0GPXo7PC2MyrrUtu6uu3DZqU4x0cl8iJxqOlx5pH0Gydj8nuUG0+X6wHZzPXZRe5M7/RzmvXch9koG6FQMiikNjUkK8Guz4NMBdAYUpxYkdPGBhiuELoaAcgIVzSGoZTuICpNYwAxBHk6q18G8Qd8vaeUdGdim4YvwycsSOwFE47wg/XmpCHrNJUZzzB9UD7Kn84TG82LAmCuAzpzcp0uPjsBckbXzj9Zt0hRgF80yDgUazsQkbgtd2p8KxIQ== loic", "key_options": null, "keyfile": "/home/dachary/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dachary", "validate_certs": true} 2021-11-11T20:39:19.275 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'dachary', 'ovpn': 'loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"name": "dachary", "ovpn": "loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC//ZCaEeVN/J09FtsvuZ4eEs4rTBtYoQyG+ZpuWkJLL90VyJeYrZzRsgvveKW2W9oFxCFcMy7IBhLes0TZ/X/4Ox9eUmP/eyqK8fBqT3K1F1W/uOmKLFhlQhkxxj+p9BOMBD5trfES/QJQ6UFSYIS2WhWaitDgONjoLMR6wXZ9r7yZeQBxbmG4wGI0Cwx4Wgg+I59UbE/5iUfnX++gN3rXrPnv9V0yFeciS4HvXRFXdrIzvQFHcZMmpOOmGruHXN/ME2dZCG+kaZhZdI6kICSlOwpU7TU1lv99zjBCeEzGfFkjv041ykKdNVR3qE0k5u0AOMqxJBORL/woSCdGvrBXqju2G/2ohl3B4zD6TAve+JLfWOV5C20NFRPPbTd53JaNpV9OQqwOfVniowzSYYBdedjm9sq6YwhJt8iBXze3OA8ZO0GPXo7PC2MyrrUtu6uu3DZqU4x0cl8iJxqOlx5pH0Gydj8nuUG0+X6wHZzPXZRe5M7/RzmvXch9koG6FQMiikNjUkK8Guz4NMBdAYUpxYkdPGBhiuELoaAcgIVzSGoZTuICpNYwAxBHk6q18G8Qd8vaeUdGdim4YvwycsSOwFE47wg/XmpCHrNJUZzzB9UD7Kn84TG82LAmCuAzpzcp0uPjsBckbXzj9Zt0hRgF80yDgUazsQkbgtd2p8KxIQ== loic", "key_options": null, "keyfile": "/home/dachary/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "dachary", "validate_certs": true} 2021-11-11T20:39:19.484 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'amctagga', 'ovpn': 'amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "amctagga", "ovpn": "amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDwVh9KqjV0b+65aBxQV66IUwNo3D5zTbN8U6Kh6eE4QtaIVe5Mmvi2SBbGMSWofE0zIIw2JmsI5LzdOoCyZl/3gNDPHQd6D9o0xRHQv94giZ2a5K+mW79foAxsuppTjV/Fbg8EZ/19W91+BqMWotlMR66ka0RPT/6U8V5mzXoydSminViO/B6q7ZYzIIgGZqaTsSxCMvz5BAzzeEaGfSlGxw5MOye9sF/tDppuRKG4rMUZULEWGuxgXAtabtXCduocAArqAY3BRJEgj7JetPdPNjipGrULyYQmT7GeknvP/waC0sPm5o8t7taJca7Vu9qKKMKPAqiK4KhlrW/yVBFliT2CjAdVuYvGj87LCcEn6NMeF+4ONwDz++YwjrAWQT2zHUJPvLnYOZPiD1LUm3F34cEHD1y4Uf+oeiYgriUMauskfFn+ShbTmWWooFRFyNobzweg9aMBZSkA+kFFifqOV1FwHTOYg342NumRGB8svK0vvTUID7AU1Ev8YXYqCEGbb0UHth/1pEbAF103enLTtPZ6qKua74rE0qJ89OF9dZXv4JY9+8Juyps7bi2XEyHNm1nHZOY5tqdY0eKFmSGJMoWBDzGgdMjmXlWp4jnY2fC619nZqxMrCQpeT70CqnBhtBemUuu6jHYUbbrPjnXIkH3FZ7/qvZt0jF0IMP9FGw== amctagga@redhat.com", "key_options": null, "keyfile": "/home/amctagga/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "amctagga", "validate_certs": true} 2021-11-11T20:39:19.515 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'amctagga', 'ovpn': 'amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "amctagga", "ovpn": "amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDwVh9KqjV0b+65aBxQV66IUwNo3D5zTbN8U6Kh6eE4QtaIVe5Mmvi2SBbGMSWofE0zIIw2JmsI5LzdOoCyZl/3gNDPHQd6D9o0xRHQv94giZ2a5K+mW79foAxsuppTjV/Fbg8EZ/19W91+BqMWotlMR66ka0RPT/6U8V5mzXoydSminViO/B6q7ZYzIIgGZqaTsSxCMvz5BAzzeEaGfSlGxw5MOye9sF/tDppuRKG4rMUZULEWGuxgXAtabtXCduocAArqAY3BRJEgj7JetPdPNjipGrULyYQmT7GeknvP/waC0sPm5o8t7taJca7Vu9qKKMKPAqiK4KhlrW/yVBFliT2CjAdVuYvGj87LCcEn6NMeF+4ONwDz++YwjrAWQT2zHUJPvLnYOZPiD1LUm3F34cEHD1y4Uf+oeiYgriUMauskfFn+ShbTmWWooFRFyNobzweg9aMBZSkA+kFFifqOV1FwHTOYg342NumRGB8svK0vvTUID7AU1Ev8YXYqCEGbb0UHth/1pEbAF103enLTtPZ6qKua74rE0qJ89OF9dZXv4JY9+8Juyps7bi2XEyHNm1nHZOY5tqdY0eKFmSGJMoWBDzGgdMjmXlWp4jnY2fC619nZqxMrCQpeT70CqnBhtBemUuu6jHYUbbrPjnXIkH3FZ7/qvZt0jF0IMP9FGw== amctagga@redhat.com", "key_options": null, "keyfile": "/home/amctagga/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "amctagga", "validate_certs": true} 2021-11-11T20:39:19.713 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'kevinz', 'ovpn': 'kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kevinz", "ovpn": "kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDtiHTDuQk8EDH5R+JfDMucI4ZpOzKN5SLD+j35QLLySgGXholoDxtqjHnmRjM1rpj5SIaXjHBZdJi7iax+1MRywHYBi5BfTtzYPgA+NH8mt+XkcdkEuFPcYxGh6fMQDQWb2g9Euze/6qi0W9mbYXvvb1/Q3bhVCXg1UgKD42BndWEpbMYFfzJLZWPtOD/lg81pwUmf/4BpPFv52sy87lQVfkkvcUliLObR4S0TI5hPLYxBS+RXmHYKgvPMB2QJ9wum62lJ+YM1MxPygS0OWbIB6UQGXmvxRXA4osXkxJ8ksPPCupOgFNiiYro91cQh3mmi6iHc9kMz5R1aNGGOfDdlnLx9DRSIiNcacjXEVNFaoM0x441f4GeU2ZR8/7uq2dfk7QjPlh1eJGxG9pzRiQ8JCDF1car0eYwXdMTsQoQqpIi5imqcELdcqGGp9FhzFElgUwpf15JYlkouVhS6hq4LDjhLNk7dDT8fmCEnHOPcbxaTgN7J3idnJK6zeIniuuk= root@iZj6ce071s2zz3reioxn93Z", "key_options": null, "keyfile": "/home/kevinz/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kevinz", "validate_certs": true} 2021-11-11T20:39:19.744 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'kevinz', 'ovpn': 'kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "kevinz", "ovpn": "kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDtiHTDuQk8EDH5R+JfDMucI4ZpOzKN5SLD+j35QLLySgGXholoDxtqjHnmRjM1rpj5SIaXjHBZdJi7iax+1MRywHYBi5BfTtzYPgA+NH8mt+XkcdkEuFPcYxGh6fMQDQWb2g9Euze/6qi0W9mbYXvvb1/Q3bhVCXg1UgKD42BndWEpbMYFfzJLZWPtOD/lg81pwUmf/4BpPFv52sy87lQVfkkvcUliLObR4S0TI5hPLYxBS+RXmHYKgvPMB2QJ9wum62lJ+YM1MxPygS0OWbIB6UQGXmvxRXA4osXkxJ8ksPPCupOgFNiiYro91cQh3mmi6iHc9kMz5R1aNGGOfDdlnLx9DRSIiNcacjXEVNFaoM0x441f4GeU2ZR8/7uq2dfk7QjPlh1eJGxG9pzRiQ8JCDF1car0eYwXdMTsQoQqpIi5imqcELdcqGGp9FhzFElgUwpf15JYlkouVhS6hq4LDjhLNk7dDT8fmCEnHOPcbxaTgN7J3idnJK6zeIniuuk= root@iZj6ce071s2zz3reioxn93Z", "key_options": null, "keyfile": "/home/kevinz/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "kevinz", "validate_certs": true} 2021-11-11T20:39:19.941 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'liu-chunmei', 'ovpn': 'liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "liu-chunmei", "ovpn": "liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1YAaxAvlRgu72huAggHpoDKSgtF/b00ACfUX4Wk5NyLFbYI3PAQapHNcNL4mIsH8pc6YkdjcJ5N/F12aEupJGQ0Q06OS324xapnaBoUy1hQl9IvhtPqTLEwR0X5xLIAhNBLIEJDNVg13MKwr0u5YVUXdrQRu7Kj9RXAPBahNZFo/Ed+cnR3LICQWliHeHFrk/DU2DUHol3a5wAPxrowAG0AxAecP6/7rASMmitvLQwdQcfIJAhWYgfyXzudg+gOmAEgv3O3U4QOikgmkm/EabUXLJnlTj6yuhtuuctdJr73UT3wLGAGmphLKCkug27kb1nTNoDa+PW/+grP63zhU1 chunmei@otccldstore05", "key_options": null, "keyfile": "/home/liu-chunmei/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "liu-chunmei", "validate_certs": true} 2021-11-11T20:39:19.973 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'liu-chunmei', 'ovpn': 'liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "liu-chunmei", "ovpn": "liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1YAaxAvlRgu72huAggHpoDKSgtF/b00ACfUX4Wk5NyLFbYI3PAQapHNcNL4mIsH8pc6YkdjcJ5N/F12aEupJGQ0Q06OS324xapnaBoUy1hQl9IvhtPqTLEwR0X5xLIAhNBLIEJDNVg13MKwr0u5YVUXdrQRu7Kj9RXAPBahNZFo/Ed+cnR3LICQWliHeHFrk/DU2DUHol3a5wAPxrowAG0AxAecP6/7rASMmitvLQwdQcfIJAhWYgfyXzudg+gOmAEgv3O3U4QOikgmkm/EabUXLJnlTj6yuhtuuctdJr73UT3wLGAGmphLKCkug27kb1nTNoDa+PW/+grP63zhU1 chunmei@otccldstore05", "key_options": null, "keyfile": "/home/liu-chunmei/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "liu-chunmei", "validate_certs": true} 2021-11-11T20:39:20.180 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'cyx1231st', 'ovpn': 'cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "cyx1231st", "ovpn": "cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLbPdDb+XHOD2fo8g3RacATj0cG07uELwUvqQP35y+JmhNRSb/ttaysFLA5ss0y77TiXx22saP3ECEm5SnuPg5IERkPQmXiw84ATTNfLnzx2G5H223SLaTQiGfSgniBCJHVPCJfNahJiurlZoOAlwDz6SKXTUiivVrFJ22iTBDCQUO6HFQKjE7aM3RAo5mZXKVlTXzM6WOd8vlzk3HMzV0/91JR6H86Lt9MEzXT4dOzqmtxd8gS6U+vA+50zOIspdK+orTRJ0PO2Fn8c1RSYM8g7uqQtpDcOEHderW8TRsqBbDxpyslK6hRX5urUNBX506XE6JNoViLNvCofgseCGj yingxin@yingxin-server", "key_options": null, "keyfile": "/home/cyx1231st/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "cyx1231st", "validate_certs": true} 2021-11-11T20:39:20.201 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'cyx1231st', 'ovpn': 'cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "cyx1231st", "ovpn": "cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLbPdDb+XHOD2fo8g3RacATj0cG07uELwUvqQP35y+JmhNRSb/ttaysFLA5ss0y77TiXx22saP3ECEm5SnuPg5IERkPQmXiw84ATTNfLnzx2G5H223SLaTQiGfSgniBCJHVPCJfNahJiurlZoOAlwDz6SKXTUiivVrFJ22iTBDCQUO6HFQKjE7aM3RAo5mZXKVlTXzM6WOd8vlzk3HMzV0/91JR6H86Lt9MEzXT4dOzqmtxd8gS6U+vA+50zOIspdK+orTRJ0PO2Fn8c1RSYM8g7uqQtpDcOEHderW8TRsqBbDxpyslK6hRX5urUNBX506XE6JNoViLNvCofgseCGj yingxin@yingxin-server", "key_options": null, "keyfile": "/home/cyx1231st/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "cyx1231st", "validate_certs": true} 2021-11-11T20:39:20.408 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'sunnagar', 'ovpn': 'sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sunnagar", "ovpn": "sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDdqw+8an6CHqsagFqXfuzQi1zCFHCh0e9pDtYz+ylA7L+7vnJLjTR7km0bb4Z6VTUh7h0/nUfF6faPzD2KGmTKvS+mTpeOUaN/H8O8j8qqh90f8z4T8F8L3DwqQbPf/R0izXmTgz0FZObFHmj+G8a+D4ndxF80EPzKkML/G0Hg2GEVdYzazSY4imSY2f56ul6qByBknhFshaTo2DUDcdbDDokF6lO6WgXhY0MJrT24CUSMyZmyi+3hF7vowW2dfbJ+SuDft4XmPDnIxRhDWWvi12jZb8Le7SWYzUlu7fAGbWe63VJqNh992myI61aZshApDD24+1xqIoo+Z/XkQD3l sunnagar@localhost.localdomain", "key_options": null, "keyfile": "/home/sunnagar/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sunnagar", "validate_certs": true} 2021-11-11T20:39:20.440 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'sunnagar', 'ovpn': 'sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sunnagar", "ovpn": "sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDdqw+8an6CHqsagFqXfuzQi1zCFHCh0e9pDtYz+ylA7L+7vnJLjTR7km0bb4Z6VTUh7h0/nUfF6faPzD2KGmTKvS+mTpeOUaN/H8O8j8qqh90f8z4T8F8L3DwqQbPf/R0izXmTgz0FZObFHmj+G8a+D4ndxF80EPzKkML/G0Hg2GEVdYzazSY4imSY2f56ul6qByBknhFshaTo2DUDcdbDDokF6lO6WgXhY0MJrT24CUSMyZmyi+3hF7vowW2dfbJ+SuDft4XmPDnIxRhDWWvi12jZb8Le7SWYzUlu7fAGbWe63VJqNh992myI61aZshApDD24+1xqIoo+Z/XkQD3l sunnagar@localhost.localdomain", "key_options": null, "keyfile": "/home/sunnagar/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sunnagar", "validate_certs": true} 2021-11-11T20:39:20.637 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'lflores', 'ovpn': 'lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "lflores", "ovpn": "lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDTlsh9TFwLCeeD2RR+56VhRZ0hp9CHj72U3Z1gbMpxpcg6PYEDzb+0qPVQp1/cIhcwwy0sm13yFl/A5i0iCRpyqB2MbyaXjj0HgkNinxLBFkPSiW2C83lfAyAhG/gLk+FcmWE4+xVrE29aoSM0JtcNtp+q83UGdnFcvhFe0pgJbt5vITo3ZMJND/UZrzeewyK8Vy4Rvpfy8aHYi2Eq+7WQrRHwUwxn0CwsxldnUfmZtu58d+Rtt78LRF0mvcScNlC7q5D+8htmYwkWkK639w5IB3xPv+Z5cV49Psi24Q+9BIr9Zo9/OnnCTauAi5ivDlh+kk/iBksAE8rQpMhvfuSFnNoq2O87aPQvNQHdRCcOa7FVIDcuocu6AB8UP4cGcJreT4+BW5Zbk6vQeJZhmj9yExpRj9MuONy5793nKuAMPn9lOjuXRfK7A5Tb3OOhpEJK1/MjtdZz4mNmFFfFIZ24hgTZO0pOiMwDf/tQ6BG7H8NS9g3QmXPyZajUocw6Szs= lflores@redhat.com", "key_options": null, "keyfile": "/home/lflores/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "lflores", "validate_certs": true} 2021-11-11T20:39:20.668 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'lflores', 'ovpn': 'lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "lflores", "ovpn": "lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDTlsh9TFwLCeeD2RR+56VhRZ0hp9CHj72U3Z1gbMpxpcg6PYEDzb+0qPVQp1/cIhcwwy0sm13yFl/A5i0iCRpyqB2MbyaXjj0HgkNinxLBFkPSiW2C83lfAyAhG/gLk+FcmWE4+xVrE29aoSM0JtcNtp+q83UGdnFcvhFe0pgJbt5vITo3ZMJND/UZrzeewyK8Vy4Rvpfy8aHYi2Eq+7WQrRHwUwxn0CwsxldnUfmZtu58d+Rtt78LRF0mvcScNlC7q5D+8htmYwkWkK639w5IB3xPv+Z5cV49Psi24Q+9BIr9Zo9/OnnCTauAi5ivDlh+kk/iBksAE8rQpMhvfuSFnNoq2O87aPQvNQHdRCcOa7FVIDcuocu6AB8UP4cGcJreT4+BW5Zbk6vQeJZhmj9yExpRj9MuONy5793nKuAMPn9lOjuXRfK7A5Tb3OOhpEJK1/MjtdZz4mNmFFfFIZ24hgTZO0pOiMwDf/tQ6BG7H8NS9g3QmXPyZajUocw6Szs= lflores@redhat.com", "key_options": null, "keyfile": "/home/lflores/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "lflores", "validate_certs": true} 2021-11-11T20:39:20.875 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'jsawaya', 'ovpn': 'jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jsawaya", "ovpn": "jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46"}, "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGq2g8OFN5msjIn5CNB2BRss9eIyJRk3r+7RQcnBOQid jsawaya@redhat.com", "key_options": null, "keyfile": "/home/jsawaya/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jsawaya", "validate_certs": true} 2021-11-11T20:39:20.907 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'jsawaya', 'ovpn': 'jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "jsawaya", "ovpn": "jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46"}, "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGq2g8OFN5msjIn5CNB2BRss9eIyJRk3r+7RQcnBOQid jsawaya@redhat.com", "key_options": null, "keyfile": "/home/jsawaya/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "jsawaya", "validate_certs": true} 2021-11-11T20:39:21.103 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'markw8973240', 'ovpn': 'shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "markw8973240", "ovpn": "shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNIdsy+yA0QjwkvjzTiH4lKB+txlh4oGkgC0r1l9swmFvD+MuzNNkD+oB/YscT51tJXsP5/wZDZMuk60xds4Os2ejoIBvzdGBgZ1G2B33etHPmzgTFE7ZegBkMlvWYmrJ9YS+iaMvXn0MMtqfE4zq0kemVAA4NBfFZMhHPJMKCg7Gj31MR0S0b7dcPx7Jnpg4A45oHr0qnufeqZYQaHGgo+LO2xdoV4V2EswWr7YG+b1wI6DzFZBbNsGyQd+cE1BzQ8NmETPNNpDPk5dCyBS7U+4ZTf/WbWk3+Cn+Ky47iWfxcscupQRkCjmTYI/uCMhnrpGU0zEHkbFwLTS6wqDtDmVHm++xof3xSsGJ2cE/vZySZGHqdxqVXfT8gt17h35rME8eX3TU392rjfK6kjEUD3UgwJJp4NFzQIEXi2HTmDYx/Vi9nuId3V1NrDz98QLPUd1ijiy1DuzI0BJPE/DTDmsb7hNXNIcAV8VRc1243W3eaee/Pdg/wmwfkrgjTU3s= shuaiwang@Shuais-MacBook-Pro.local", "key_options": null, "keyfile": "/home/markw8973240/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "markw8973240", "validate_certs": true} 2021-11-11T20:39:21.135 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'markw8973240', 'ovpn': 'shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "markw8973240", "ovpn": "shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNIdsy+yA0QjwkvjzTiH4lKB+txlh4oGkgC0r1l9swmFvD+MuzNNkD+oB/YscT51tJXsP5/wZDZMuk60xds4Os2ejoIBvzdGBgZ1G2B33etHPmzgTFE7ZegBkMlvWYmrJ9YS+iaMvXn0MMtqfE4zq0kemVAA4NBfFZMhHPJMKCg7Gj31MR0S0b7dcPx7Jnpg4A45oHr0qnufeqZYQaHGgo+LO2xdoV4V2EswWr7YG+b1wI6DzFZBbNsGyQd+cE1BzQ8NmETPNNpDPk5dCyBS7U+4ZTf/WbWk3+Cn+Ky47iWfxcscupQRkCjmTYI/uCMhnrpGU0zEHkbFwLTS6wqDtDmVHm++xof3xSsGJ2cE/vZySZGHqdxqVXfT8gt17h35rME8eX3TU392rjfK6kjEUD3UgwJJp4NFzQIEXi2HTmDYx/Vi9nuId3V1NrDz98QLPUd1ijiy1DuzI0BJPE/DTDmsb7hNXNIcAV8VRc1243W3eaee/Pdg/wmwfkrgjTU3s= shuaiwang@Shuais-MacBook-Pro.local", "key_options": null, "keyfile": "/home/markw8973240/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "markw8973240", "validate_certs": true} 2021-11-11T20:39:21.332 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'luisrico', 'ovpn': 'lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "luisrico", "ovpn": "lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+ZvZuogU+WfyGj4BR2xYllNoQqAoxQ6AM0IYddBV2pL1xgTM/SopuAxDRP0q2stpozMtjVjlvuiIgr/caqyTuOgYtYCvB09ppefqdnCDrJJBCOKpjUEnCpAKcBMMuz9V32Zqt5FgpIi0Mu7Gp/QCvB97imxEkoUunpdX/8Ri/xz0nJO+QqoJX5wOn91jG49XnkMCLzx+MuafhT90WFO9CV1sIVREZS+X2C7ACQuhx9Qwk7R4BXt2f3M4dA0K2nLwaAnVW50M1y6Gs50Wl+ojDR1itySOxv8YpwyRlRTOzQTyVMnG0Qsaxxwp+hy4b/vQW3NIUvHYdTyvqALTQdGHT lricomor@lricomor-mac", "key_options": null, "keyfile": "/home/luisrico/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "luisrico", "validate_certs": true} 2021-11-11T20:39:21.363 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'luisrico', 'ovpn': 'lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "luisrico", "ovpn": "lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+ZvZuogU+WfyGj4BR2xYllNoQqAoxQ6AM0IYddBV2pL1xgTM/SopuAxDRP0q2stpozMtjVjlvuiIgr/caqyTuOgYtYCvB09ppefqdnCDrJJBCOKpjUEnCpAKcBMMuz9V32Zqt5FgpIi0Mu7Gp/QCvB97imxEkoUunpdX/8Ri/xz0nJO+QqoJX5wOn91jG49XnkMCLzx+MuafhT90WFO9CV1sIVREZS+X2C7ACQuhx9Qwk7R4BXt2f3M4dA0K2nLwaAnVW50M1y6Gs50Wl+ojDR1itySOxv8YpwyRlRTOzQTyVMnG0Qsaxxwp+hy4b/vQW3NIUvHYdTyvqALTQdGHT lricomor@lricomor-mac", "key_options": null, "keyfile": "/home/luisrico/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "luisrico", "validate_certs": true} 2021-11-11T20:39:21.570 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'matan', 'ovpn': 'matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "matan", "ovpn": "matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n"}, "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKjxTgyhL1yFriyNvvuhUc6SE8+6f6lT6FTF8cZL9uiz matan.brz@gmail.com\nssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM8l7qvSQR7fnZdQ1yB1F1jgde0OmLcwcDFzc/o2dYBb matan.brz@gmail.com", "key_options": null, "keyfile": "/home/matan/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "matan", "validate_certs": true} 2021-11-11T20:39:21.592 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'matan', 'ovpn': 'matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "matan", "ovpn": "matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n"}, "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKjxTgyhL1yFriyNvvuhUc6SE8+6f6lT6FTF8cZL9uiz matan.brz@gmail.com\nssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM8l7qvSQR7fnZdQ1yB1F1jgde0OmLcwcDFzc/o2dYBb matan.brz@gmail.com", "key_options": null, "keyfile": "/home/matan/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "matan", "validate_certs": true} 2021-11-11T20:39:21.799 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'ozeneva', 'ovpn': 'ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ozeneva", "ovpn": "ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkFTEUxQkQ1cplOLOf86Teu4qTYGkMqWtjIwzsU/AcimjFolSI5fJ3B72PH7bhQKkSnOlTjiOos2LrD5i/XgxkJD6icVWwHO11lczsq/8K5x0ZMJbJaiT3QphBB4yItoge2Y4/7g4o/78Ne+U7eZ1L5fgeO1fExtIaEXK1lS8pVlpZQXyproTIzp1WLHoaXb38Mxc5jAtuvEKfEycdX3c8k/9ti2qBCZnmjPH05WOxkqCRyNBT+bSJZyFMYt9RDUuycn3BYVSXTAzPW0qwgvKY5vij+qiFoxij7nKNMH7w6C0arolCcJxhiMyEF3UbTuOxgVONWyqFk/vScGGSYj9ytQYjv0L1cFyOVhlpbu+tnptulfQ3faMpjROE/eYxTIoY7OPDgWbFqEUJ4eEhKwAC7aYIYUhvoI/8k+4u64nsyEJQdHXmB+OY3WE5TegYUCuqSsdPY74eO1o8dDgxF363eggDSBmYqSSs+Tpu9TNhJG7iJFX1tiAPwmKvjVrJOac= ozeneva@omri", "key_options": null, "keyfile": "/home/ozeneva/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ozeneva", "validate_certs": true} 2021-11-11T20:39:21.820 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'ozeneva', 'ovpn': 'ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ozeneva", "ovpn": "ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkFTEUxQkQ1cplOLOf86Teu4qTYGkMqWtjIwzsU/AcimjFolSI5fJ3B72PH7bhQKkSnOlTjiOos2LrD5i/XgxkJD6icVWwHO11lczsq/8K5x0ZMJbJaiT3QphBB4yItoge2Y4/7g4o/78Ne+U7eZ1L5fgeO1fExtIaEXK1lS8pVlpZQXyproTIzp1WLHoaXb38Mxc5jAtuvEKfEycdX3c8k/9ti2qBCZnmjPH05WOxkqCRyNBT+bSJZyFMYt9RDUuycn3BYVSXTAzPW0qwgvKY5vij+qiFoxij7nKNMH7w6C0arolCcJxhiMyEF3UbTuOxgVONWyqFk/vScGGSYj9ytQYjv0L1cFyOVhlpbu+tnptulfQ3faMpjROE/eYxTIoY7OPDgWbFqEUJ4eEhKwAC7aYIYUhvoI/8k+4u64nsyEJQdHXmB+OY3WE5TegYUCuqSsdPY74eO1o8dDgxF363eggDSBmYqSSs+Tpu9TNhJG7iJFX1tiAPwmKvjVrJOac= ozeneva@omri", "key_options": null, "keyfile": "/home/ozeneva/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ozeneva", "validate_certs": true} 2021-11-11T20:39:22.027 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'sudhanaruna', 'ovpn': 'ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sudhanaruna", "ovpn": "ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDepyzA+MeQT5oRM52SxmUUKVAzkcrH/NqFxoCpBAjUSioTMaB1h0PhHE00nvHnKRvqWdPl8LCBWS3Ncjh30r9lwHaP+oPy4iLQEc7D2Mwn7WyPh5cq9N3UF25C0AlMG744k1tBNb97qy73OJI+oc5pAj4vUpCC1Ut9pMP5E5pQCx/kwgSOpC7mk0dYr/H5FxjPb0ca55eisFTLXraHPSChdrHjlYC5XpOZ9csilTwFdF0fJdu0nMDwIxGvwTOaUcIcfNLkpGB2Sj/pAM4L9pVsIbeE3C8d978MZ8Ff+ksQ0T8eNzew2ZjSkSuV+oR1sBCJYZ7mbDQuYLKAs5DanO1VFJvgjBJm1zP2ONu+g0fn/j04qIosuudziwoG/dOTkFo+6dBvF5hQi7TbZOIUVFr5dBlGS+O4eoK0gi/wpzun227E4dadY9uB1wiZ/PTD/5og+gxWG32RkHqqX3ALlK4m8JKzrq214nZ7Dt9CNewiUkR15uRkDasGIdiu6qb4Ez8= ArunaSudhan@sudhan-linux", "key_options": null, "keyfile": "/home/sudhanaruna/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sudhanaruna", "validate_certs": true} 2021-11-11T20:39:22.049 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'sudhanaruna', 'ovpn': 'ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "sudhanaruna", "ovpn": "ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDepyzA+MeQT5oRM52SxmUUKVAzkcrH/NqFxoCpBAjUSioTMaB1h0PhHE00nvHnKRvqWdPl8LCBWS3Ncjh30r9lwHaP+oPy4iLQEc7D2Mwn7WyPh5cq9N3UF25C0AlMG744k1tBNb97qy73OJI+oc5pAj4vUpCC1Ut9pMP5E5pQCx/kwgSOpC7mk0dYr/H5FxjPb0ca55eisFTLXraHPSChdrHjlYC5XpOZ9csilTwFdF0fJdu0nMDwIxGvwTOaUcIcfNLkpGB2Sj/pAM4L9pVsIbeE3C8d978MZ8Ff+ksQ0T8eNzew2ZjSkSuV+oR1sBCJYZ7mbDQuYLKAs5DanO1VFJvgjBJm1zP2ONu+g0fn/j04qIosuudziwoG/dOTkFo+6dBvF5hQi7TbZOIUVFr5dBlGS+O4eoK0gi/wpzun227E4dadY9uB1wiZ/PTD/5og+gxWG32RkHqqX3ALlK4m8JKzrq214nZ7Dt9CNewiUkR15uRkDasGIdiu6qb4Ez8= ArunaSudhan@sudhan-linux", "key_options": null, "keyfile": "/home/sudhanaruna/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "sudhanaruna", "validate_certs": true} 2021-11-11T20:39:22.256 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'zl31wang', 'ovpn': 'zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "zl31wang", "ovpn": "zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDuHIpWAz6Sk3ATLvTG4KS24JUzgqQBEm4EesPgf1cEG+xK4gISUixaVC9OSNS/KR+8/yoWAhmw992p8cp7mWXxqrRC9kIrthLWvD6Jjz3HTXgarO2oNibeVu27kK/Fm2vvUNACARBkoC5YXd6IuoZOEg1bJu06DLw5Y7pGMqZRVDOrE8WUDwr27roZSgU2/N+18Y1LYn4gwSvWpuC/IotLyGSpW7/dgrZAIreRoYboMdTSlIjjuzeWvYx/AnZmkeEo0IhfyuBRl7ZQwxSFYHn9Z1+iQkt+O9hEZYSZuYRuyFPRqHNMKCEj1iYVRMcwbf/x2/dhjfmbWmjOMJoNNwm+DpixHHbId6Idx82GlAtCgQSTe9aG783oCeZYuVUr1plIRjeEsdjY9UBRYKNknEAChKjMSbxysvO3Dy4yrU2r8qo1iBS1fi4HUg12asCLrJX7vSCvOhOn7ltqDTmt5UiLSn+eCzXpOTBDWz7McB28FoYyR+Rwz5GOH3ITFhPyhXc= wzl@manjaro-wzl", "key_options": null, "keyfile": "/home/zl31wang/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "zl31wang", "validate_certs": true} 2021-11-11T20:39:22.288 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'zl31wang', 'ovpn': 'zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "zl31wang", "ovpn": "zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDuHIpWAz6Sk3ATLvTG4KS24JUzgqQBEm4EesPgf1cEG+xK4gISUixaVC9OSNS/KR+8/yoWAhmw992p8cp7mWXxqrRC9kIrthLWvD6Jjz3HTXgarO2oNibeVu27kK/Fm2vvUNACARBkoC5YXd6IuoZOEg1bJu06DLw5Y7pGMqZRVDOrE8WUDwr27roZSgU2/N+18Y1LYn4gwSvWpuC/IotLyGSpW7/dgrZAIreRoYboMdTSlIjjuzeWvYx/AnZmkeEo0IhfyuBRl7ZQwxSFYHn9Z1+iQkt+O9hEZYSZuYRuyFPRqHNMKCEj1iYVRMcwbf/x2/dhjfmbWmjOMJoNNwm+DpixHHbId6Idx82GlAtCgQSTe9aG783oCeZYuVUr1plIRjeEsdjY9UBRYKNknEAChKjMSbxysvO3Dy4yrU2r8qo1iBS1fi4HUg12asCLrJX7vSCvOhOn7ltqDTmt5UiLSn+eCzXpOTBDWz7McB28FoYyR+Rwz5GOH3ITFhPyhXc= wzl@manjaro-wzl", "key_options": null, "keyfile": "/home/zl31wang/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "zl31wang", "validate_certs": true} 2021-11-11T20:39:22.495 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'mayagilad', 'ovpn': 'mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mayagilad", "ovpn": "mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDIi8JMOVD91QR7EXBkhxeAsB2UO2gyWqYPiP6HjJO/TzzMKf7QtO928XzdORwQ1/jm2hDF1kaNPaU/XIHD0hHSvfICS4ELQcBPa5uPya6d5Kn5fVlpGqBcXBxKGbHB01PGjn0214eupt2LebWxkm7TP/BGDHKXXaoR748AlbyyG2L3TaNoGkwQiEoZe3CWgH4qkrf2CEpyfOa7BM7U0h+lceJxIGe+Yo6Ll14zyngDBa059E+ndFTpkQ6vECII7FdB1+A4Orhu40XHZgyVnlRudD1l80pIVnZves8N2zHZhQo8hp0sB5oY8HENiB09j7Xa6YQ/c1c2wm9E2xJHSNo7muFiHz48vQEgvNmu0lZdbVwXIy3IgLEBHjs6Kyqjtg618V4UaoP/LBOqtWtLd6TvGJ3qBlNyvhO5gZ2DGeLMN2WWoHxfl3VM+QGCGI2dnzUyHhfG1VBDYj8fzWXOuUVpVCMUw8fEg8GWmYLDwm3SbdMACWU87J0kAmaDPKcwwL6AwtWJZr/p89sSypl6y50JlOoDrbDbxrIbHQgbf1tcEgPOuLUN+O06qpi+OxKMtFdbvz+NQ4V4CpXpvMMQZj5AbuHp0EJ5Vme/Kcmpl/xoYGyFDzOEGR8lw6LMz+CpQ4GtnK5XXe1Xn3TJh6Tn2tVyS20p7artemGQQquCN8iQOw== mayag@gett.com", "key_options": null, "keyfile": "/home/mayagilad/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mayagilad", "validate_certs": true} 2021-11-11T20:39:22.517 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'mayagilad', 'ovpn': 'mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "mayagilad", "ovpn": "mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDIi8JMOVD91QR7EXBkhxeAsB2UO2gyWqYPiP6HjJO/TzzMKf7QtO928XzdORwQ1/jm2hDF1kaNPaU/XIHD0hHSvfICS4ELQcBPa5uPya6d5Kn5fVlpGqBcXBxKGbHB01PGjn0214eupt2LebWxkm7TP/BGDHKXXaoR748AlbyyG2L3TaNoGkwQiEoZe3CWgH4qkrf2CEpyfOa7BM7U0h+lceJxIGe+Yo6Ll14zyngDBa059E+ndFTpkQ6vECII7FdB1+A4Orhu40XHZgyVnlRudD1l80pIVnZves8N2zHZhQo8hp0sB5oY8HENiB09j7Xa6YQ/c1c2wm9E2xJHSNo7muFiHz48vQEgvNmu0lZdbVwXIy3IgLEBHjs6Kyqjtg618V4UaoP/LBOqtWtLd6TvGJ3qBlNyvhO5gZ2DGeLMN2WWoHxfl3VM+QGCGI2dnzUyHhfG1VBDYj8fzWXOuUVpVCMUw8fEg8GWmYLDwm3SbdMACWU87J0kAmaDPKcwwL6AwtWJZr/p89sSypl6y50JlOoDrbDbxrIbHQgbf1tcEgPOuLUN+O06qpi+OxKMtFdbvz+NQ4V4CpXpvMMQZj5AbuHp0EJ5Vme/Kcmpl/xoYGyFDzOEGR8lw6LMz+CpQ4GtnK5XXe1Xn3TJh6Tn2tVyS20p7artemGQQquCN8iQOw== mayag@gett.com", "key_options": null, "keyfile": "/home/mayagilad/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "mayagilad", "validate_certs": true} 2021-11-11T20:39:22.725 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'arjsharm', 'ovpn': 'arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "arjsharm", "ovpn": "arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDFG1CoPZEh4KBANdJZIzLsutHsF6tAYZTdDdEW+5E+8YyNHOy42D8iBLg+eCAbIl4SFmJTsPNtyO4L8/Ut79tkG3usA9nuGoAwGYuctZUTgOiTiV61940Y+NTvwan9vFNoknVi/3qIGNSXdm90I1aWRTKLCXRbGA1qvu+qyJv93O8LFaPni2TJmtAebY/xf0fHhW2fURUoeoCsoobrTkPA8SojT0A3SM+31iOD7sxbmsqLBR790wz/FHh152PiqpupIijS9TaZgGuVpejM3rgAC7H2kVmKJlrmDlmfcbKLS6BBi+uUDi6ta8whnDRq03nP/4LJE7ZDIxX3I4mHztZeJggURlsfpJdo99+VHkeWlGDBIP41QgD4UX1neuMcIO3nuEnZmLeJyvo9JOR/I7AudbQCDPjSMNbrF+el70mWLhvPe33NA3x6eOBRPc3r+qrSwMhXEPatZlbywJRY+O6yC2TYqdlUKsUEVA/D0Iw7tO+pf3yH3Y/iUDJJLvViRZs= Arjun@localhost.localdomain", "key_options": null, "keyfile": "/home/arjsharm/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "arjsharm", "validate_certs": true} 2021-11-11T20:39:22.746 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'arjsharm', 'ovpn': 'arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "arjsharm", "ovpn": "arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDFG1CoPZEh4KBANdJZIzLsutHsF6tAYZTdDdEW+5E+8YyNHOy42D8iBLg+eCAbIl4SFmJTsPNtyO4L8/Ut79tkG3usA9nuGoAwGYuctZUTgOiTiV61940Y+NTvwan9vFNoknVi/3qIGNSXdm90I1aWRTKLCXRbGA1qvu+qyJv93O8LFaPni2TJmtAebY/xf0fHhW2fURUoeoCsoobrTkPA8SojT0A3SM+31iOD7sxbmsqLBR790wz/FHh152PiqpupIijS9TaZgGuVpejM3rgAC7H2kVmKJlrmDlmfcbKLS6BBi+uUDi6ta8whnDRq03nP/4LJE7ZDIxX3I4mHztZeJggURlsfpJdo99+VHkeWlGDBIP41QgD4UX1neuMcIO3nuEnZmLeJyvo9JOR/I7AudbQCDPjSMNbrF+el70mWLhvPe33NA3x6eOBRPc3r+qrSwMhXEPatZlbywJRY+O6yC2TYqdlUKsUEVA/D0Iw7tO+pf3yH3Y/iUDJJLvViRZs= Arjun@localhost.localdomain", "key_options": null, "keyfile": "/home/arjsharm/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "arjsharm", "validate_certs": true} 2021-11-11T20:39:22.954 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'melissali', 'ovpn': 'melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "melissali", "ovpn": "melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCXZDAUNEK4+/6Y+ywEtFnabnfvZmzzHhO6y4PbRHgx8SoQ8+7au/5ElB0vjjiJJSChnDymdqP0UzwcShzhWA8is6K9c6tXUHg6NcgV5tN2xYYmGIIWwm66Z1kYH6u1x2Jkv9LO04rXgebkzXPPYK401agCWjkjUv5acZE12+M/B6d4h2lpFjzBqZ/mmRNp0tHdCapdG/nS1i6GRi3Yi3fQ89CbiZArG9268DSEnLe+npeyBMVdDkWpVIytQ6pU1+g0prrePIVaMvrK3yEooduD4HeLr4liqkINj6rCs90xedPcFi0++UaLFRcgWh3OOnrI952Oe19dx5hlbmcnMPGjaXnkng2keWFyZMOQRRti5l8C/TXPUTaYGWEoqFEEabIKnCura0NWuZHpeF9xEK0tnvNVeauuUfkPxWQ7PH8QQ/9XjOkKbmy0+2q9tug8RC5i5NwcRE4W3ijM7pLwhWMdxkmIOe3w1taPq8HU95hMQchl2749oj8DRyVePVK9aH0= melissali@melissali-ubuntu", "key_options": null, "keyfile": "/home/melissali/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "melissali", "validate_certs": true} 2021-11-11T20:39:22.986 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'melissali', 'ovpn': 'melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "melissali", "ovpn": "melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCXZDAUNEK4+/6Y+ywEtFnabnfvZmzzHhO6y4PbRHgx8SoQ8+7au/5ElB0vjjiJJSChnDymdqP0UzwcShzhWA8is6K9c6tXUHg6NcgV5tN2xYYmGIIWwm66Z1kYH6u1x2Jkv9LO04rXgebkzXPPYK401agCWjkjUv5acZE12+M/B6d4h2lpFjzBqZ/mmRNp0tHdCapdG/nS1i6GRi3Yi3fQ89CbiZArG9268DSEnLe+npeyBMVdDkWpVIytQ6pU1+g0prrePIVaMvrK3yEooduD4HeLr4liqkINj6rCs90xedPcFi0++UaLFRcgWh3OOnrI952Oe19dx5hlbmcnMPGjaXnkng2keWFyZMOQRRti5l8C/TXPUTaYGWEoqFEEabIKnCura0NWuZHpeF9xEK0tnvNVeauuUfkPxWQ7PH8QQ/9XjOkKbmy0+2q9tug8RC5i5NwcRE4W3ijM7pLwhWMdxkmIOe3w1taPq8HU95hMQchl2749oj8DRyVePVK9aH0= melissali@melissali-ubuntu", "key_options": null, "keyfile": "/home/melissali/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "melissali", "validate_certs": true} 2021-11-11T20:39:23.188 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'ionutbalutoiu', 'ovpn': 'ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ionutbalutoiu", "ovpn": "ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCYdmNB3np8OLc49B4cAh59CdNYIb4fAq2XBQ1sbOZMMZmhpFobyy+i6s4lBnovMNWAB6HmpsECHrKRU4IFl9lQ7vV+++J08SLoAkl6G0Gk9tmbgEs+/ovCZg+qUfZA1ZccSAPHmtqfZOiJcnlIlhQq2BoFnLUwmpeIQ4H5D+Y2q7DMdyGkdCenH80aD5KuiwLIyiBL9q7RYz0U5cdCO3wDFU4+5XuZiT8VYDUgp3mWPpAiPdGsJmCn41tpt3PViPDQlKh4vLW0ieQUJI/v2f6oHbM9YkxSoVlwLcrjTv+Nl+dVymqVTF4UaT1kz+ZnoH6khKZxSHvKm9YHc4LNZKYoVdyCxdZvBfxbEbXpT702vLqDNGEfZSlzJs5kVqP30m8fI/yzTQeD7tgjzzrKWdLZacyo+9cWeH6nFfZAQuZ9ug1lv3UAs5x6MsidEH5u6Iymi8BE+mUsOXnSpSfUDDo5n6+CMVe93brdW21iSX9Hbuz5RKB6xile1ROHpvpi1u7VQ9Hp3FIZFFiix9S5jniyB2OqjGKTO6RULfPsJo8sXCLq9YKq9zrkpMv5wURMQKnjtS5OHUWh0cde3U/VikPIfLkzEsfLV8ICAksyuWHQJGVUMxhPmSTKtRsPScbklekT+n65rexDt4Qr73YOy9BUIPV4Mo+kkl4ugQd6VCKJ6w== ibalutoiu@cloudbasesolutions.com", "key_options": null, "keyfile": "/home/ionutbalutoiu/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ionutbalutoiu", "validate_certs": true} 2021-11-11T20:39:23.219 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'ionutbalutoiu', 'ovpn': 'ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "ionutbalutoiu", "ovpn": "ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCYdmNB3np8OLc49B4cAh59CdNYIb4fAq2XBQ1sbOZMMZmhpFobyy+i6s4lBnovMNWAB6HmpsECHrKRU4IFl9lQ7vV+++J08SLoAkl6G0Gk9tmbgEs+/ovCZg+qUfZA1ZccSAPHmtqfZOiJcnlIlhQq2BoFnLUwmpeIQ4H5D+Y2q7DMdyGkdCenH80aD5KuiwLIyiBL9q7RYz0U5cdCO3wDFU4+5XuZiT8VYDUgp3mWPpAiPdGsJmCn41tpt3PViPDQlKh4vLW0ieQUJI/v2f6oHbM9YkxSoVlwLcrjTv+Nl+dVymqVTF4UaT1kz+ZnoH6khKZxSHvKm9YHc4LNZKYoVdyCxdZvBfxbEbXpT702vLqDNGEfZSlzJs5kVqP30m8fI/yzTQeD7tgjzzrKWdLZacyo+9cWeH6nFfZAQuZ9ug1lv3UAs5x6MsidEH5u6Iymi8BE+mUsOXnSpSfUDDo5n6+CMVe93brdW21iSX9Hbuz5RKB6xile1ROHpvpi1u7VQ9Hp3FIZFFiix9S5jniyB2OqjGKTO6RULfPsJo8sXCLq9YKq9zrkpMv5wURMQKnjtS5OHUWh0cde3U/VikPIfLkzEsfLV8ICAksyuWHQJGVUMxhPmSTKtRsPScbklekT+n65rexDt4Qr73YOy9BUIPV4Mo+kkl4ugQd6VCKJ6w== ibalutoiu@cloudbasesolutions.com", "key_options": null, "keyfile": "/home/ionutbalutoiu/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ionutbalutoiu", "validate_certs": true} 2021-11-11T20:39:23.417 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'pdiazbou', 'ovpn': 'pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pdiazbou", "ovpn": "pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDuyLB5rnyjlglcHVCoCSG7pDVJQcTYq/I2yGr3ohK9p6boWxk+zW/a2Q7+qNdKk2DtUC4WvvZ76BU56FC6/LPIUKSJLbN/Xv9/dnZcqKf+ZVlCQgMrE8yGoWt8WLtejJ1iBj0EdoDfYwDiL09ygUim1L1hNxFDOVq9RA2Lo6Tqk5lcAgjoCnU4NZY2YzTXOUSZTzsv/1bd6WVudKHJuWF6ZPb+mbbMoWqwnUZCHwjW8PgTXaPhk1929P0ftWT4ncfKnCc46LXf8OkvzQgCrLyh8964tGKPWppp3wNg2AJ/zFvRY8kkJZSxOawp9Gu8+kCNwWUmLvE9bqYJKT7EMDrU+8TMUEWLFl5fGoxoLWD113PPBcFrpUhvRAJa625eec8a/inv4cqPf6dY6qyXAReXURJm7LnHT4fDq+F6op8+tmZT6Tzkw4+YMUd9h0XDJ4pnPQpk5tjWER/HuWm73+ialXo+ERQWnnVCB1IXq1FEOeF8YAslKC7aSYIZjG9uUkM= peristocles@localhost.localdomain", "key_options": null, "keyfile": "/home/pdiazbou/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pdiazbou", "validate_certs": true} 2021-11-11T20:39:23.449 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'pdiazbou', 'ovpn': 'pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "pdiazbou", "ovpn": "pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDuyLB5rnyjlglcHVCoCSG7pDVJQcTYq/I2yGr3ohK9p6boWxk+zW/a2Q7+qNdKk2DtUC4WvvZ76BU56FC6/LPIUKSJLbN/Xv9/dnZcqKf+ZVlCQgMrE8yGoWt8WLtejJ1iBj0EdoDfYwDiL09ygUim1L1hNxFDOVq9RA2Lo6Tqk5lcAgjoCnU4NZY2YzTXOUSZTzsv/1bd6WVudKHJuWF6ZPb+mbbMoWqwnUZCHwjW8PgTXaPhk1929P0ftWT4ncfKnCc46LXf8OkvzQgCrLyh8964tGKPWppp3wNg2AJ/zFvRY8kkJZSxOawp9Gu8+kCNwWUmLvE9bqYJKT7EMDrU+8TMUEWLFl5fGoxoLWD113PPBcFrpUhvRAJa625eec8a/inv4cqPf6dY6qyXAReXURJm7LnHT4fDq+F6op8+tmZT6Tzkw4+YMUd9h0XDJ4pnPQpk5tjWER/HuWm73+ialXo+ERQWnnVCB1IXq1FEOeF8YAslKC7aSYIZjG9uUkM= peristocles@localhost.localdomain", "key_options": null, "keyfile": "/home/pdiazbou/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "pdiazbou", "validate_certs": true} 2021-11-11T20:39:23.647 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'xuwang', 'ovpn': 'xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "xuwang", "ovpn": "xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2iBYL614KRYvFhow408yzMljYRdfyMmbnkUVN8waja4RCsmxTpvIU1aBQ02bMTsY/0n+WMWAbI1D6C1HbuG+JbFYBLSgPCTSG8FIpukFU8y1by8piPu2/ZBD1XdFcPO6u1Nr/tZC08EXhfqvnDAVJUyg6rLLs5+TFKAiPSuv5GyrMRWvMI8eoPk+4u2r/ThxXZTDaIfJhPXlULMLL5ZxR4KZU+WtWbCGSFDGjSugCqdTzpZyX5jUq9C9JB83RxojCtI5n9rCoXLcGOvW9HBnVMxhS56zE8BwxLxLhGy6S1lOoyBPzu7w6XIc160JjCK5iAmlXRqG93nWEVrhZkxCz wangxu298@foxmail.com", "key_options": null, "keyfile": "/home/xuwang/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "xuwang", "validate_certs": true} 2021-11-11T20:39:23.679 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'xuwang', 'ovpn': 'xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "xuwang", "ovpn": "xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2iBYL614KRYvFhow408yzMljYRdfyMmbnkUVN8waja4RCsmxTpvIU1aBQ02bMTsY/0n+WMWAbI1D6C1HbuG+JbFYBLSgPCTSG8FIpukFU8y1by8piPu2/ZBD1XdFcPO6u1Nr/tZC08EXhfqvnDAVJUyg6rLLs5+TFKAiPSuv5GyrMRWvMI8eoPk+4u2r/ThxXZTDaIfJhPXlULMLL5ZxR4KZU+WtWbCGSFDGjSugCqdTzpZyX5jUq9C9JB83RxojCtI5n9rCoXLcGOvW9HBnVMxhS56zE8BwxLxLhGy6S1lOoyBPzu7w6XIc160JjCK5iAmlXRqG93nWEVrhZkxCz wangxu298@foxmail.com", "key_options": null, "keyfile": "/home/xuwang/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "xuwang", "validate_certs": true} 2021-11-11T20:39:23.886 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'blaine', 'ovpn': 'blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "blaine", "ovpn": "blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDKSdu3rI8o3qId+GU5KYsxB9CLPTODJFiwy+ww6SgrNKgPfiaSUsZAvcQwoxHfuaEXj6VGaOS6HyFFmvHKhzbecSQCzPIucIuKJJDA7VTaA9Jet0aT2b8Vk4p0Ud9j4WoausmxEdBQC+ZpJxSuqajplejYK1Yi3YEqXSCvM6Md76HCustS8efPvzLsLYttDd34Lw0/zp7ALE8hrepLYJgzQS5Z6btvQEjL7lvtHk80/Y+M6ZIQAl6nPcjb8u+wfs/lxcRA5i94GsZhEoYAeMQptJNvOaA5vYK4QtNyV6lyjGVJl4RUUAtLKKp4JE/kdx7sa0Gy99L+u4MjzBoP5B7SkdJnJx7FioXEen5l5DpiJ6xRixw+SjbqEffW4HHI5+1k6jP6kuHPbWfcrw693+aiFVDQA+myzfWo7iDTjY2H5CtnCh+wr1t3VktXhoyPob/XfMPzMNg4W8nA9KVePwzEXSy2Ro0t71GiFQqBviUuqfktqz4a52PqGyFHtskhFAH3H/h6JYM88vfc7Bw7P/u4eVaoCNgNIbLA/kmf689Q1H2KENj2SX7CDwNL9GCOy/pg89qai63+PFT8258zTi0TywWduP1D0DXXtIB3e5K1yK4Zt6VCinXZAze4bsT+z8y4f9W2Xs/FhhtUmo3mOiDJuaGmuWRzHQQTjL//v/Rimw== blaine@brgardne-mac", "key_options": null, "keyfile": "/home/blaine/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "blaine", "validate_certs": true} 2021-11-11T20:39:23.908 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'blaine', 'ovpn': 'blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "blaine", "ovpn": "blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDKSdu3rI8o3qId+GU5KYsxB9CLPTODJFiwy+ww6SgrNKgPfiaSUsZAvcQwoxHfuaEXj6VGaOS6HyFFmvHKhzbecSQCzPIucIuKJJDA7VTaA9Jet0aT2b8Vk4p0Ud9j4WoausmxEdBQC+ZpJxSuqajplejYK1Yi3YEqXSCvM6Md76HCustS8efPvzLsLYttDd34Lw0/zp7ALE8hrepLYJgzQS5Z6btvQEjL7lvtHk80/Y+M6ZIQAl6nPcjb8u+wfs/lxcRA5i94GsZhEoYAeMQptJNvOaA5vYK4QtNyV6lyjGVJl4RUUAtLKKp4JE/kdx7sa0Gy99L+u4MjzBoP5B7SkdJnJx7FioXEen5l5DpiJ6xRixw+SjbqEffW4HHI5+1k6jP6kuHPbWfcrw693+aiFVDQA+myzfWo7iDTjY2H5CtnCh+wr1t3VktXhoyPob/XfMPzMNg4W8nA9KVePwzEXSy2Ro0t71GiFQqBviUuqfktqz4a52PqGyFHtskhFAH3H/h6JYM88vfc7Bw7P/u4eVaoCNgNIbLA/kmf689Q1H2KENj2SX7CDwNL9GCOy/pg89qai63+PFT8258zTi0TywWduP1D0DXXtIB3e5K1yK4Zt6VCinXZAze4bsT+z8y4f9W2Xs/FhhtUmo3mOiDJuaGmuWRzHQQTjL//v/Rimw== blaine@brgardne-mac", "key_options": null, "keyfile": "/home/blaine/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "blaine", "validate_certs": true} 2021-11-11T20:39:24.116 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'name': 'choffman', 'ovpn': 'choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "choffman", "ovpn": "choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDM7LPHiaPQXdE+yt9cn7UMuRr8k/LjSUmMrnrAqp8ySQtX6OA3/NcOsxg6FJ45chZqeoXwhsYEW+7pTInDqFlOlmVynR26lWnZfk9DjBJZcloUCn8y5MR9QitNYsOMs5KwCEFmBH6sP+2vCFocXBnKQcVZLRfip3MEG4LoWYfI4ir7fLmK1Hr5G+GhtMstXvVSG4VE4IgbU9/LRIOv7P9qFn4bRJoUSiS2dDbUxVIuneA9bIEnB9ZEVGahEzlUhx6qDSs0FvrnneCQ1ayOFpxClzjT5GuKk/J3LJvWAw8CAwDXCf3FzsWQsvb5ru5VSazUI9uA3o9wWkbLW2oZPgtzI8lmwbL+MT58T92tqaJOPID3IZDGjzNrRqU5BQYpvFYHpHRRiqV625og+ktYfXgO+dTbgAxScomqutydI4cyJusDtnWtljWnEocS+2hUSZpmB6B3Wj70TuuzsbYrcrJ/NgRpQNX4dwHCJl6vPT+xkoLh1imYAC7klf7cnrOyGcs= choffman@choffman.net", "key_options": null, "keyfile": "/home/choffman/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "choffman", "validate_certs": true} 2021-11-11T20:39:24.316 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'name': 'choffman', 'ovpn': 'choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c'}) => {"ansible_loop_var": "item", "changed": true, "comment": null, "exclusive": false, "follow": false, "item": {"name": "choffman", "ovpn": "choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDM7LPHiaPQXdE+yt9cn7UMuRr8k/LjSUmMrnrAqp8ySQtX6OA3/NcOsxg6FJ45chZqeoXwhsYEW+7pTInDqFlOlmVynR26lWnZfk9DjBJZcloUCn8y5MR9QitNYsOMs5KwCEFmBH6sP+2vCFocXBnKQcVZLRfip3MEG4LoWYfI4ir7fLmK1Hr5G+GhtMstXvVSG4VE4IgbU9/LRIOv7P9qFn4bRJoUSiS2dDbUxVIuneA9bIEnB9ZEVGahEzlUhx6qDSs0FvrnneCQ1ayOFpxClzjT5GuKk/J3LJvWAw8CAwDXCf3FzsWQsvb5ru5VSazUI9uA3o9wWkbLW2oZPgtzI8lmwbL+MT58T92tqaJOPID3IZDGjzNrRqU5BQYpvFYHpHRRiqV625og+ktYfXgO+dTbgAxScomqutydI4cyJusDtnWtljWnEocS+2hUSZpmB6B3Wj70TuuzsbYrcrJ/NgRpQNX4dwHCJl6vPT+xkoLh1imYAC7klf7cnrOyGcs= choffman@choffman.net", "key_options": null, "keyfile": "/home/choffman/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "choffman", "validate_certs": true} 2021-11-11T20:39:24.385 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:24 +0000 (0:00:35.552) 0:01:44.641 ***** 2021-11-11T20:39:24.406 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:24 +0000 (0:00:00.021) 0:01:44.662 ***** 2021-11-11T20:39:24.469 INFO:teuthology.task.ansible.out: TASK [users : Update authorized_keys for each user with literal keys] ********** 2021-11-11T20:39:24.471 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'andrewschoen', 'ovpn': 'andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "andrewschoen", "ovpn": "andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.480 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'zack', 'ovpn': 'zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "zack", "ovpn": "zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.482 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'kdreyer', 'ovpn': 'kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kdreyer", "ovpn": "kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.492 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'dmick', 'ovpn': 'dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dmick", "ovpn": "dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.493 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'andrewschoen', 'ovpn': 'andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "andrewschoen", "ovpn": "andrewschoen@home hYDlX2i7X8t0JHpK2smvFA 2a6e753fb3c401ad75e1273ae5e18e4abc0404149dbdc3675bb317a63a4469e9"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.494 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'sage', 'ovpn': 'sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sage", "ovpn": "sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.505 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'zack', 'ovpn': 'zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "zack", "ovpn": "zack@ceph0 1CeZnO8st4CuhLpH/DJNVw 7230fda728e34998da4401284fc3e5c8939bfdd52482f8cb675d8acf6fcd12b4\nzack@mba.local mzf7/v4tv3uVExt16N/w/Q 13dcf7082b974315dfab1c76885dfc094d7ccaf01ae2e7898a20005602be2caa\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.506 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'dgalloway', 'ovpn': 'dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dgalloway", "ovpn": "dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'kdreyer', 'ovpn': 'kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kdreyer", "ovpn": "kdreyer@redhat.com yUBHWOGnPts8xuDSon5ODg 38feb4fec9f1ba3de74ed074d745ed46a8e5b3ac1d44e50005a04cf07f0462cd"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'akraitma', 'ovpn': 'akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "akraitma", "ovpn": "akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.517 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'dmick', 'ovpn': 'dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dmick", "ovpn": "dmick@angus z8fyazJ0pc56I/XTlHAOrQ db682b1b89c790bf26a2e12dbea03f4495f8ea5f248e29b2f5c5dc072acb3c93\ndmick@home o9fblAdxhBToEJxbutv2mw 1704b16322c3984ed565b48cce52877eadade26b4093443b349b70f846f0eac6\ndmick@dell r6meTomZVVH/WVOmVf2aSQ 8ac1a9d0452166af59894c6b57ed07a12b42042aa6d2618df2bec9c962684072\ndmick@dmick-MacBookAir EFN6/5V1I5YNm72XjlApUg f2335a3fa49a0296233ab5faba28e7bbf309a89c193169dbe3c411e4219827a4\ndmick@hplaptop 49Wf5LFRBuobkfWp7OoAdQ 1401efaf459e0b7673f03b546eeff2c8144219c674eb3c94aa23e5af5ffc66b9\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.519 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'jcaratzas', 'ovpn': 'jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jcaratzas", "ovpn": "jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'sage', 'ovpn': 'sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sage", "ovpn": "sage@autriche 2Uhfc/1jXje4W7G4s0sJEQ 09631754d059fab96e16be441b3b8f7e63b37a260677897a2669cf7a2501d2d3\nsage@gnit YuLp2iihT6cCqY+GmW9IVQ dc79b19c45b9be1b4e69cd7debeb5b87aaa8cd0050353f798913da22c233875a\nsage@maetl wpsQIdtAb+z+ietqr+90Zw 42c518015b7cd440e7c8afd65fcca5769e31e9ef8f8a67d109a3de38187087e6\nsage@lowride tak6ngfj0ohW6upHRUMq7g 450919c617f452e874ef2a8cfa5ca18f47d249450ef0b0374849e721b0c53a70\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.529 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'trociny', 'ovpn': 'mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "trociny", "ovpn": "mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.531 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'dgalloway', 'ovpn': 'dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dgalloway", "ovpn": "dgallowa@laptop POebac0SJIVkSCyi0LE4yg 52fc0cb34f3332a5f3cff117a20f00e8ea89a051736cb543a689a96dd9528ae5\ndgalloway@p50 7PHdbcNE/YjcHOm5NbNfpQ 7075211c35997b01797c2f15083aedf13888fc4c54080130342dccc76469fb40\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'akraitma', 'ovpn': 'akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "akraitma", "ovpn": "akraitma@aklap 2zvxeeQ+bQ4UhKBE9Kz2DQ ff4a9dbab841ee62e7328d8cd0d285c68c22a1de6e23ea24f470b3dcb1a27045"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.541 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'jcaratzas', 'ovpn': 'jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jcaratzas", "ovpn": "jcaratza@desktop QI358a0dizGe6T42hCYj1A 139ab306be981d0a669ec2288a2529503882deffceb28bdfc9bde2b70a825898"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.552 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'trociny', 'ovpn': 'mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "trociny", "ovpn": "mgolub@mirantis.com +WMPxNCgiMwhNJAJHNCV2g 4cd6749ac740ea6dfb50d4f602920ac75e979286439bd510540a8a551728104e"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.763 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'smithfarm', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf', 'ovpn': 'smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf", "name": "smithfarm", "ovpn": "smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf", "key_options": null, "keyfile": "/home/smithfarm/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "smithfarm", "validate_certs": true} 2021-11-11T20:39:24.774 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'mbenjamin', 'ovpn': 'mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mbenjamin", "ovpn": "mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.776 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'aemerson', 'ovpn': 'aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "aemerson", "ovpn": "aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.786 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'smithfarm', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf', 'ovpn': 'smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf", "name": "smithfarm", "ovpn": "smithfarm@wilbur vEk7281ao7DkpXE2Cn3rpw 20058c04c4991caf38b9517faedb75c8562e8d8669808b6be9f70da6a5b64007"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6jqQtqO7pnW39mpSQQC4L2FuxPOx/5j34lp2PrwuTWJoDwtUVCZhpNwSs5UAj3wdXGXCzWYWxu9d9Gh0UEBkiC+2ZHtLvWrwS1TDVRy5g5jYVm9vX3kL5dhoC2vYPyQoQkxeMX3BYlhLwkSjCAnjaceJJZyOT7KsqbpanvJxvudMzeyCosntnMuxyPzjH9CkVKqycddZnscdHMkDIZ3TjNhnL8GKK/QCPB60tkCO6HWN/g4HxidRjrp2VjQgIa5MWN9LBR6mGspZI6+rR1ygLhbl1scr1cCCumh65r+zmCC/ZY33YCfRXdmQZo+ShsBa+KbcvakEx0Bqgx7DsMoiR ncutler@pantograf", "key_options": null, "keyfile": "/home/smithfarm/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "smithfarm", "validate_certs": true} 2021-11-11T20:39:24.788 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'mbenjamin', 'ovpn': 'mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mbenjamin", "ovpn": "mbenjamin@ferret N7Ln6KHUAYT/iiP4Swkb0A 244b90574d7e294f6f80949ef5e640bab103c2c34e1902a69cdcf592607d950c\nmbenjamin@duel PRRFsQiomHSJX8sVgRHblA f3009adb7502a6c5554e1157d36c63d84f40fe5074a01d7d0280e5c48da374dc\nmbenjamin@rael 8ixfNNx3j2m2+nGhXkHsnQ 62fda7ec724f0fdb3f05add4c3739e07a5c7eeb2380d5f1f7eeeb13db35e16a8\nmbenjamin@shatner hJ1oEl5wSZaKPJC1Ugss8g 9c9161b53da4d842308f80e5b490a0c1539076f129bd0e34ee2253272dd55c21\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:24.797 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'aemerson', 'ovpn': 'aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "aemerson", "ovpn": "aemerson@seasalt enqtMOCQ0a24hizRK46SZg 42f36c9c14c1eb7c468e41cf4c5649e30037f0aaf5eefbd6c07be9637224ca01\naemerson@electric TjQxreDKF6bsIehyQnX4aQ a73b7d56b95ba3d4dda5ded995b9cb98eaaf5d2db7d148c340aa4f4c68cf26d7\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.006 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'amaredia', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour', 'ovpn': 'ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour", "name": "amaredia", "ovpn": "ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour", "key_options": null, "keyfile": "/home/amaredia/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "amaredia", "validate_certs": true} 2021-11-11T20:39:25.017 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'amaredia', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour', 'ovpn': 'ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour", "name": "amaredia", "ovpn": "ali@freerunner yQjRpRVG7D5KN2HAUjI30g 9d677a34ae98477e6cc8ba1d975d81dcae43a102013b265c63f3ea91e7dacd78"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCc1ZbHNXuJn7gxyt7QJ579gEM13MuFdTzsrnedYuRIW2Zlm4rCFr8Oj5SGs5DWBIJnd3W4O2v7PjuVQMxU/zbwnj7mdwBmLFe1cSzOJv2eP1R2uaU5z5C7KNmyPLU++pGKClzb6v5wcEQUq2K35xnuXUU9B935dK+Fm7bK7+HAxj+1vpVeycbPFyPhf6mwbx8dZv4uvZGV2+CGBuyIB/5U2AMJZy9LWim3AR35bip4ftXvSKlAON+RHhnS0toG/6uwp0XlFuGn5H8snaca7L6hGtB4xg1PqA5aMf33Jiv2NVLQo8emHU9J/HeNVS7ksoSZ6InynpLZ6b9uXa9OM9XL ali@parkour", "key_options": null, "keyfile": "/home/amaredia/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "amaredia", "validate_certs": true} 2021-11-11T20:39:25.236 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'name': 'tserlin', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet', 'ovpn': 'tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet", "name": "tserlin", "ovpn": "tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet", "key_options": null, "keyfile": "/home/tserlin/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "tserlin", "validate_certs": true} 2021-11-11T20:39:25.238 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'dis', 'ovpn': 'dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dis", "ovpn": "dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.247 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'gregf', 'ovpn': 'gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "gregf", "ovpn": "gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.249 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'joshd', 'ovpn': 'jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "joshd", "ovpn": "jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.259 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'name': 'tserlin', 'key': 'ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet', 'ovpn': 'tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5'}) => {"ansible_loop_var": "item", "changed": false, "comment": null, "exclusive": false, "follow": false, "item": {"key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet", "name": "tserlin", "ovpn": "tserlin@annarbor DlKe+OWBPcFAQtWMUAHnwg 6b268bd737ffa5dd38865575ccd444b92cb912c70f5b82dac41f9c50505df4a5"}, "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2ok6CBUpOVGv2RFws44GtEP5SVxOi4Vie0WSZYLpD55rTfmOtsItN1d1EciVNTUyWuyzMeQGWC4JAd3/2l3gR/5ZwSvd7b/7TCNYxjAMEubRand0GxEoiKhpkJMMmJqcT0KefP8pr31MASWPuERj1+0/IbjJExsvrJaUjqeIfZ+DWR8dC2VYdcH3hsp6AE3mqKX/9693sxe8ROt6qY4WkpZcO4M90unOVa2CnJsYqKaaIC4z3fmKuHZpJZjiJMrg8rtuN4r7bnKWPEVGcahj+i74JWwKR5+2gntLpxw2chIBmf4qFu6HDplddig4V3I/2NLB8soBpgc+m8O7YyYl0w== thomas@easystreet", "key_options": null, "keyfile": "/home/tserlin/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "tserlin", "validate_certs": true} 2021-11-11T20:39:25.260 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'cmeno', 'ovpn': 'cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "cmeno", "ovpn": "cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.271 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'dis', 'ovpn': 'dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dis", "ovpn": "dis@zambezi wXYUFvWPBlkCFz+mC4RD6A 11c464dfb2a27986e029f1915732a4f237baba4eade02bb045c8f0d13dfada28"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.272 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'ivancich', 'ovpn': 'ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ivancich", "ovpn": "ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'gregf', 'ovpn': 'gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "gregf", "ovpn": "gregf@kai YhNrPfedhZjbvGjOfmotOA 440cf8595a87cd307790bbf79c3668455a0405945e2b271d873325de222cd72f\ngregf@pudgy VZrk8nWzg7pYLOrZru8dBg c1d1a0e469a134ccf5f5a5525631a6b83efa6970beec3b23809eb0daa3dca47f\ngfarnum@WorkMini2 +bAqcTdU7Ok9bGMcB3A84w 3cff1326561a23cf81dd6495373cb83ed149cee026c6374d72c19b483f4f1f07\ngfarnum@Macbook bxVtolCC9SY3QNlpx3cE1w aff8d28bfb4d693253511d29e8d399196e964fc096594ec705748a5469d44654\ngregf@fedoragreg Jdn8I/sFGcr5Aa/dici6lw 50f88afc35c05ef8454742226f7baf2cd20cb1e2d4d0c9f4a393013877736bfa\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'yuriw', 'ovpn': 'yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48 2021-11-11T20:39:25.274 INFO:teuthology.task.ansible.out:\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "yuriw", "ovpn": "yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'joshd', 'ovpn': 'jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "joshd", "ovpn": "jdurgin@glarthnir ElGaAgbs5VZujuzsQMfCmA 2a156febba037d02d1099bc11d1e697d34300b2c420f2df664b5b0de1248f983\njdurgin@new-angeles jqa015PRJcHSp5WHcwJjUg 42113e1156382fde866d691f30584f6b30c3dfc21317ae89b4267efb177d982c\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'tamil', 'ovpn': 'tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "tamil", "ovpn": "tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06 2021-11-11T20:39:25.276 INFO:teuthology.task.ansible.out:b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.284 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'cmeno', 'ovpn': 'cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "cmeno", "ovpn": "cmeno@boxtop KD0XJIGun8xcnNI83pqrsg 79079fe78e1cf401cccb14692c9d16f63e2010c57cfbf56c487387affca90aed"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.286 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'jowilkin', 'ovpn': 'jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jowilkin", "ovpn": "jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.296 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'bhubbard', 'ovpn': 'brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "bhubbard", "ovpn": "brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.298 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'ivancich', 'ovpn': 'ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ivancich", "ovpn": "ivancich@ann.arbor Kt2vxZ3Ge609mHfjx0W4Cw aaa55a9e2b5276b62a21cd3c401b365c5c2693e39efccb2f9edefafefa1dc8b1"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'yehudasa', 'ovpn': 'yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "yehudasa", "ovpn": "yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'yuriw', 'ovpn': 'yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3 2021-11-11T20:39:25.299 INFO:teuthology.task.ansible.out:e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "yuriw", "ovpn": "yuriweinstein@Yuris-MacBook-Pro.local wKA8mCcsdsk/CE+1d9GiPw caaf5bf5eb31ef269e3d0bc34d45dd761c0bb0907172eea6e754434de4603db7\nyuriw@home 02TZyR3JHJMxEQob80ICNA 85b4aa0f69f6d469dae0bb3dca4baaf222e164927ed7eed2082caae8f4717e48\nyuriweinstein@xenon1 C9eVdLb/i18lMcMG20rGPw eaddd0e831a77de3f35cb19e307bd6f9aeb0cc03eff1ae58490d7db33ced4311\nyuriw@yuriw-RH2 WhMAZ+zZ/PPXOc+4HEDaZg 2d3e28b31edc3f2be43f6b73150dedc9c9df6e316d29afc1416d6b96a982e217\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.308 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'dang', 'ovpn': 'dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dang", "ovpn": "dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.310 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'tamil', 'ovpn': 'tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "tamil", "ovpn": "tmuthamizhan@mac /1CBLC6rxqzJzPspffZV0g 4d1dfff2e097a7fc2a83ea73eccad2f0e453a6338e18c87b4d89bf370733e29c\ntamil@tamil-VirtualBox M22QdmhkSPj9aEcTiuIVfg 8e76e06b14de99318441c75a96e635a92f5bddc54a40b45276191f6829c6b239\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'branto', 'ovpn': 'branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "branto", "ovpn": "branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'jowilkin', 'ovpn': 'jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jowilkin", "ovpn": "jowilkin@jowilkin 2r8xOv/eCTcHr+HSsMkPYg 0ac416d2dc139628144dfa0822af8cc0a455f5f5f3e4d1b9713c14115c062218\njohn@osd-host 7zjDTxAYhCmTX+Az4SJaoA 7d924233fdef168e2c5c01258aa349de108629ef2ff90d17c0b96acf22dac7c2\njohn@admin-host 7cpk7iJ1Hg2vk4bPDovKmA 05765178f27af6dc4e43e47f52d773aac3bc1b3f1dd998bdbf479b951bfd2efb\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.320 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'xiaoxichen', 'ovpn': 'xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "xiaoxichen", "ovpn": "xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.322 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'bhubbard', 'ovpn': 'brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "bhubbard", "ovpn": "brad@dhcp-1-165.bne.redhat.com 4oShQI9+vYtX5gA47np/Sw 3fc7df5afa772752d8eee15c01d550cc1dcc88b6e940abc9f9f8f26102d239d4"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'ffilz', 'ovpn': 'ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ffilz", "ovpn": "ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'yehudasa', 'ovpn': 'yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "yehudasa", "ovpn": "yehudasa@yehuda-2 NGXHeO0lAvRFfYLalffzEw 0d3489b09f48ad5fa9d1ec5944d9c8020daf852488f389f3d44fe63d3f651f34\nyehuda@yehuda-940X3G shisK5LjI6fr3ZBJy/xX8A 49522899bd26130086ce668079f0062d987d85dfa5767dd5c34e5953db97997a\nyehudasa@yehudasa-desktop OT1MhoO0WihhvkKztqW0Uw 12a4d6b54390b9df7f5af3bd6b533f3c1fee0c7b9fbb79f0a87bcb28b182c7d4\nyehuda@crestview yUJ7nsAXRv78MgIyxJ93Yw 5b872feb20cbd895a5d8c87956f9cbd491471654b1a29aa8c13826cd95a44c40\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.333 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'joao', 'ovpn': 'joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "joao", "ovpn": "joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.334 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'dang', 'ovpn': 'dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dang", "ovpn": "dang@sidious w0CNW2g9K1WiRenVGYWNUA 4f59d761bfab3659115da2b3b80a486266f77b09d8527983217d15648b4f92b4"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.344 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'nhm', 'ovpn': 'nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "nhm", "ovpn": "nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.346 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'branto', 'ovpn': 'branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "branto", "ovpn": "branto@work ganye+HpG3dkMEik6WtTng 018f3f9b9d49dcefa701ea304a8e58f002c46f0650edae220a0a7ab1bce36aeb"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'mkidd', 'ovpn': 'linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mkidd", "ovpn": "linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'xiaoxichen', 'ovpn': 'xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "xiaoxichen", "ovpn": "xiaoxichen@ebay RvJJ7BhIehpoPtggrwnskQ 862ecfe7e15dfab86d61df86856bfe06cbb99f240f6f03851f7f9e1a255327d6"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.356 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'jlopez', 'ovpn': 'JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jlopez", "ovpn": "JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.358 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'ffilz', 'ovpn': 'ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ffilz", "ovpn": "ffilz@redhat.com 6YdvqxkKfmDWGD2s0wA7Ww 4ce64d08686e34e559ccec2498df433b155b70c9ebccaec616b6b34f0f0c246e"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'haomaiwang', 'ovpn': 'yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "haomaiwang", "ovpn": "yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'joao', 'ovpn': 'joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "joao", "ovpn": "joao@magrathea eSS2/gvK7ALE6L9bITfuMA c3caaeecee3f43e39b7a81fad50e0d874359c70a9c41b77c661511c71f733909\njoao@timesink 9S3oER36HheVupjRpnLz6A 9dbc964184244e9da269942dc73ec9ebba6594bcccfdc0eb09562b58b4542162\njoao@tagide uUDVmZ1S5lp/Hx8eKTfDCw ee37985d0a51360f08266e0de208ea85616777a325d498d7c895a80d4723f844\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.368 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'kchai', 'ovpn': 'kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kchai", "ovpn": "kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.370 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'nhm', 'ovpn': 'nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "nhm", "ovpn": "nh@latte JCH9icAtDPj951rgbPKJyw 9a3d30ec1ec9467ccdc8bdbbfacffd7396fb38e046199ae37b2b7b69dbf37480\nnhm@espresso +YYZPT29wYzY5ooaRzabCQ 1ee041dd58b9ec6eb678c47632ece7cf6c24e23bcbac28a77a82af05ba6cc148\nnhm@mocha HgOGOfkBEzJihFsKmPRfKQ 2e17f3ba0b90df7a36f19a7c8f64d2aa8f966b2794c94caa110d313e927a1c1b\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vumrao", "ovpn": "vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'mkidd', 'ovpn': 'linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mkidd", "ovpn": "linuxkidd@zenbook oYp1WWV0JwpikHCWCV52Lg 9aca455b601bf3a365d31068154150ac63dd76f32cef29a55f9685dd1a88aa22"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.380 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'dfuller', 'ovpn': 'dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dfuller", "ovpn": "dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.382 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'jlopez', 'ovpn': 'JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jlopez", "ovpn": "JCLO@oh-mbp-jcl vZhmBh/1LjLFEu+atRec6w 1f13f591373b4dc798a9b701fabf1eb99bf4aa58f87b6420d6c916716f0965af"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'owasserm', 'ovpn': 'owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "owasserm", "ovpn": "owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.392 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'haomaiwang', 'ovpn': 'yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "haomaiwang", "ovpn": "yuyuyu101@desktop XAUS/1Geh1T2WY//5mRahw fda03bdaf79c2f39ac3ba6cd9c3a1cb2e66b842a921169f20a00481a4cd3d9cb"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.394 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'vasu', 'ovpn': 'vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vasu", "ovpn": "vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'kchai', 'ovpn': 'kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kchai", "ovpn": "kefu@gen8 HVoNrB5C8+bYxuqIfByeEQ 4dddde1890af2d6df367d3d832cc3b9b660160a1db69f0135e0d09364b2cb9b3"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'smohan', 'ovpn': 'shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "smohan", "ovpn": "shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.404 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vumrao", "ovpn": "vumrao@vumrao VaMHdnIGTl6y9LIkurxfjQ 71de53c4a0f212b8f919437d7d433d24a33d7a33bc6fe5df5d047e24499994b2"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.406 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'abhi', 'ovpn': 'abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "abhi", "ovpn": "abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'dfuller', 'ovpn': 'dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dfuller", "ovpn": "dfuller@laptop 6U0lNVidRy7Ye/5q6cGN1A aea3d019f68a95094c99385aff099818224455a829615cfc774587e4519398a7\ndfuller@rex001 6Z8bfQDgPXVSGMeeQHoItg 3abd41920b72683fbba7f25be88ff992fcd753119c4d2086c12daaf20798e684\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.417 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'cbodley', 'ovpn': 'cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "cbodley", "ovpn": "cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.418 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'owasserm', 'ovpn': 'owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "owasserm", "ovpn": "owasserm@orit-desktop zi8oe8INwK6IZrtUz6lg0w b5f4f96232a77a260f7a16244af873df2ff143c2bde0dbaabf02e9952b8f38ca\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'mwatts', 'ovpn': 'mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mwatts", "ovpn": "mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'vasu', 'ovpn': 'vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vasu", "ovpn": "vasu@ceph1 +1D1qNUAk8h7OF9LF9qkrQ 963aa256fb99bc4b77e7085f57cf910a04c7f143603f81252331411eb8e37ec3\nvakulkar@vakulkar.sjc.csb O8ac1k0Dh3xkIFX8NFyIZw 471538eeb22384b58921e4f11af272c00c0a953dc7fe8d95ba057e65d141fbd2\nvasu@vasuSrv waJqYAARY/LnfuP1x/KQzQ 68915d3a1eb3dd00a562c149791cec5f43a96f5fd0b851ec855ec3f5dab496b4\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.429 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "prsrivas", "ovpn": "pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.430 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'smohan', 'ovpn': 'shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "smohan", "ovpn": "shmohan@laptop 7wwHLZZNa4ShUV1imXDDjw 0aca19a8ff6dbeee2821dd75a329a05b8052170204b2b242ced9b1a68ca8df37"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'pdonnell', 'ovpn': 'pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pdonnell", "ovpn": "pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'abhi', 'ovpn': 'abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "abhi", "ovpn": "abhi@trusty SKarQTpBigBobP9sLjdLiw 868a74ed21b46f7f64255897d824f4e3eb21f8dde844bbdaa386681c942d8114"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.441 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'jlayton', 'ovpn': 'jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jlayton", "ovpn": "jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.443 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'cbodley', 'ovpn': 'cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "cbodley", "ovpn": "cbodley@redhat.com W4gFS4oU8PtUOrdHuBYwXQ af7dafd992687f5d85a79866838a78e5070a4934fb0a935e8094adb31ec28611"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'rzarzynski', 'ovpn': 'rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rzarzynski", "ovpn": "rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.453 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'mwatts', 'ovpn': 'mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mwatts", "ovpn": "mdw@degu IPiZqcFT2BLuf2h3+tw58g 7af390a631ec11bddd7d1ac506d29af65e1e01e19f7dc931b4f459030cb7a195"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.455 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'rdias', 'ovpn': 'rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rdias", "ovpn": "rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "prsrivas", "ovpn": "pritha@dhcp35-190.lab.eng.blr.redhat.com GCk3PIYzUNZ6/xrsKoq8VQ acbfc4279970b44c3d008990e0cf1bb5eb280299218441a0f25fda988bc555f6\nprsrivas@t480 jciDaqMz2kZbSRymwpJiqA 68baba83dd966fe9536bc9a822428710b10a28781f695bc3fb750482a94b0e8c\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.465 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'vshankar', 'ovpn': 'vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vshankar", "ovpn": "vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.466 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'pdonnell', 'ovpn': 'pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pdonnell", "ovpn": "pdonnell@redhat.com Q9j56CPPXipXScmKi57PlQ fb616603b6d27cf65bfa1da83fc0ca39399861ad1c02bfed37ce9be17cdfa8ea"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'akupczyk', 'ovpn': 'adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "akupczyk", "ovpn": "adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'jlayton', 'ovpn': 'jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jlayton", "ovpn": "jlayton@redhat aNfzMdXOhhmWRb25hwXJIg f51fee42c5268f7b8e00d57092dc522b0a07b31154ea52cf542da9cac5885868"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.477 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'nojha', 'ovpn': 'nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "nojha", "ovpn": "nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.479 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'rzarzynski', 'ovpn': 'rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rzarzynski", "ovpn": "rzarzyns@redhat.com h4UkibgFG40ygfpfKTMBrg e20ca28c60144dbabc97953cd4c273c1b92cd45ebcddd0f0299679d7a5c87d7f"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'ifed01', 'ovpn': 'ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ifed01", "ovpn": "ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'rdias', 'ovpn': 'rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rdias", "ovpn": "rdias@rdias-suse-laptop 0bh54sdB69mh95n5rWME5g 452e3338e48d04d4e816f4f1cb54d637746a7acc1ffe5e8ed4c1506c8e07a72e"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.489 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'myoungwon', 'ovpn': 'myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "myoungwon", "ovpn": "myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.491 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'vshankar', 'ovpn': 'vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vshankar", "ovpn": "vshankar@h3ckers-pride eYOkK3Yj10T5PFLIUu5DKw f6d9d714fb6a97e51a4c8d2a4fb7ed0c0e87caedfd0fa813b209846abd9a8a20"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'gabrioux', 'ovpn': 'gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "gabrioux", "ovpn": "gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'akupczyk', 'ovpn': 'adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "akupczyk", "ovpn": "adam@TP50 C0YuBT9bYaNhdDmjbF56xg 5d298b33b9dbaef364b037561aa5c5de374405bb8afead5280db5b212506ea58"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.502 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'leseb', 'ovpn': 'leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "leseb", "ovpn": "leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.504 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'nojha', 'ovpn': 'nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "nojha", "ovpn": "nojha@localhost YQTw/h6ZMgdJn7EPBmbEnw 253574eae62759c4c5d3bc4bf949c59948a0488e4dfe4c91ee754a3b5494847e\nnojha@nojha-mac aucFu11aJL4iZg9ch76Lmg 6f14a860471fabc125d167c7af8d4729c26bc93bfab5e73d5457ebf54d32d526\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'hchen', 'ovpn': 'hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "hchen", "ovpn": "hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'ifed01', 'ovpn': 'ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ifed01", "ovpn": "ifed01@snowmon CjHOevVvBm+PwhjnhZYM7g 7393d3a95b1874764fc4e7ae2d481f1859b336e16e1ba53c49d126585e11ba56"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.514 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'jcollin', 'ovpn': 'jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jcollin", "ovpn": "jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.516 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'myoungwon', 'ovpn': 'myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "myoungwon", "ovpn": "myoungwon@fg2server p5CiGtdZolEwS1dCiRJjkg cd2491f835be1474a7802db607cf9a31c59e1031001e1dfb8fe240616dda9a71"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'xxg', 'ovpn': 'xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "xxg", "ovpn": "xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'gabrioux', 'ovpn': 'gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "gabrioux", "ovpn": "gabrioux@elisheba 80kx1htp39RsFrlGONcp+A a95579ef6f90694cd6fd390302adf8532237a8ea65bd5544d9b561654d712ba2"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.527 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'pcuzner', 'ovpn': 'pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pcuzner", "ovpn": "pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.528 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'leseb', 'ovpn': 'leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "leseb", "ovpn": "leseb@mbp cCx1v5/FfaQ/IQHujqtG9Q 6121d11f9abfa6b1b36330eafaa2196249a9c92f989be25c9fac1558292c920f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.538 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'mkogan', 'ovpn': 'mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mkogan", "ovpn": "mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.540 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'hchen', 'ovpn': 'hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "hchen", "ovpn": "hchen@host12 hUr3k0rugStZMjvIxIvCOg 9d57e14d49901f18b24ee4076ae7e6a2f9eb6fd9fbce786660c448486c966fca"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vumrao", "ovpn": "vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'jcollin', 'ovpn': 'jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jcollin", "ovpn": "jcollin@earth +H4Hk4WcNuqdQj7ch/Nulw 8426545e6457c9e1e8adca2af5ddf836fbcfb433cdc5359fd135afdf4e0f7d2a\njcollin@stratocaster jbjV3FsrsTJwyKUA3Y8VVQ 0439745f795fef1399636bd550040d45445d1607b471284c5c9b9dbccc86a987\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.540 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'henrix', 'ovpn': 'henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "henrix", "ovpn": "henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.551 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'xxg', 'ovpn': 'xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "xxg", "ovpn": "xxg@zte Y2d/Ov201XMivDNwo4nUoQ 5e5da8d579793601699af628300430c1e5dd469c8bcff7c3ee11d23ec004bdcc"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.553 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'pbs1108', 'ovpn': 'pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pbs1108", "ovpn": "pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'pcuzner', 'ovpn': 'pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pcuzner", "ovpn": "pcuzner@rh460p oK28wU5DSabvEL4VjDRhEg a449ed81d7e2970f418263fb3ce10dd711d03925a0990ddf298f826aae1caa53"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.563 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'clacroix', 'ovpn': 'clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "clacroix", "ovpn": "clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.565 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'mkogan', 'ovpn': 'mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mkogan", "ovpn": "mkogan@mkP50 f/LENRJbsyepmvZA23F7Fg b908f1c0237a7ee56b73dc42f2df79b49ca83d6f4573f5229e7cfe6b4ad7b6a2"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'epuertat', 'ovpn': 'epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "epuertat", "ovpn": "epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'vumrao', 'ovpn': 'vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vumrao", "ovpn": "vumrao@redhat TMNqzMvbJS8Va/8nT9QUQw ab386c2bd7c6796d5413e4d841a16dda2504cca6d95df831a652a30d2e5655ed"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.576 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'tdehler', 'ovpn': 'tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "tdehler", "ovpn": "tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.578 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'henrix', 'ovpn': 'henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "henrix", "ovpn": "henrix@hermes iPPDBfnLzP5Pe5FuTcJBmw b26aefb8a61451066f984e074f708ea9ca6b2c5d7cca35996c08b0b2bb2c2736"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'laura', 'ovpn': 'laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "laura", "ovpn": "laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'pbs1108', 'ovpn': 'pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pbs1108", "ovpn": "pbs1108@bspark_cli0 nSCINdeTLTLEO5JP/GIwRQ 76372ad6f7ad731556ff13605c3729eacaf59dcf7f9ac82dd9a8501bd95d3b26"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.588 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'yaarit', 'ovpn': 'yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "yaarit", "ovpn": "yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.590 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'clacroix', 'ovpn': 'clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "clacroix", "ovpn": "clacroix@redhat.com ZGY9sgvjT0BuJRi9zrULzg c3311aba4025aa42cd78c999dcee4e2c16415a3ac44ac8c95b77838459ef3315"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'rpavani1998', 'ovpn': 'admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rpavani1998", "ovpn": "admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.600 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'epuertat', 'ovpn': 'epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "epuertat", "ovpn": "epuertat@private BnUoirwxGRWXtLxulJU5xA da2cfc4995bed82ef85db3633edad0a7eb2c32ba559a48259b10be94a8fdf006"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.602 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'rishabh', 'ovpn': 'rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rishabh", "ovpn": "rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'tdehler', 'ovpn': 'tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "tdehler", "ovpn": "tdehler@think 7e0WC4Vh86XWZviZ9WBMgw 4dc8477db6e4f40312e6b2b9db293dc009e49e518015ace20431c0fb69025461"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'skrah', 'ovpn': 'skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "skrah", "ovpn": "skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.612 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'laura', 'ovpn': 'laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "laura", "ovpn": "laura@flab 2DGHIAxD39eNKOPkn3M67w ab1ae304abed3824a68b5c0ecf4f92fca76a4f8b9fcbcc0ca43388a85b7f9305"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.614 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'smanjara', 'ovpn': 'smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "smanjara", "ovpn": "smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'yaarit', 'ovpn': 'yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "yaarit", "ovpn": "yaarit@centos TBuWjkAsj1GB/V9eWc/R1Q 7bd86a857dec48dc25850ecf0c00486d9a89c2ff5f88b2f28c3e36bdeb139fce"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'bengland', 'ovpn': 'bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "bengland", "ovpn": "bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.625 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'rpavani1998', 'ovpn': 'admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rpavani1998", "ovpn": "admin1@rajula 31GbDo9d1YnW5BQ8u3utvw a2da13cb840f848846023c85442ba7bcce97dc186056a0ecc036a220d7eb7fc3"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.627 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'pnawracay', 'ovpn': 'pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pnawracay", "ovpn": "pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'rishabh', 'ovpn': 'rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rishabh", "ovpn": "rishabh@p50 zdJ4XsBdVugwMrqJOSBi3Q c78bb28ba5cf2bf9c8edb80fe57814d60cd2ffdbd874cf9a271e5adf171bb0c4"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'alfonsomthd', 'ovpn': 'alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "alfonsomthd", "ovpn": "alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.637 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'skrah', 'ovpn': 'skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "skrah", "ovpn": "skrah@thinkpad IboPmnUdsLqqRXlHQ1RT5w bf85db9e916dceaf84a1e6ea33c59eb4adb424cb4e727ce0a903a3498b799ed2"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.639 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'oliveiradan', 'ovpn': 'oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "oliveiradan", "ovpn": "oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'smanjara', 'ovpn': 'smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "smanjara", "ovpn": "smanjara@fedora OfpUShUejOW2W/R4FlKQ7Q fab77ef619f730a5aaf255e4167aa7a35a7e5450d12820923d969036a6326b0f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.650 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'swagner', 'ovpn': 'swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "swagner", "ovpn": "swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.652 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'bengland', 'ovpn': 'bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "bengland", "ovpn": "bengland@bene-laptop ud1gWgoNggJTS7LQtPTZTA d3ebd084ce385cb450ce2f83c02dc66a1637dedbc7a8b191dab68acfc935af41"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'yuvalif', 'ovpn': 'ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "yuvalif", "ovpn": "ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'pnawracay', 'ovpn': 'pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pnawracay", "ovpn": "pnawracay@flab LIurEXLa7xbr+jzf2wRJVg 04062ba385602b385fd17f14de3a0cad83c685b8078fd2f18cc9ad77a4f4762d"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'kkeithle', 'ovpn': 'kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kkeithle", "ovpn": "kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.663 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'alfonsomthd', 'ovpn': 'alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "alfonsomthd", "ovpn": "alfonsomthd@localhost iG3hmp+gS/4aOL4Vp/Nj4w ef1480c222cb2d540cf932ef2f27967891b768c2ad90b59a12a5279ceaa18b47"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.664 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'emmericp', 'ovpn': 'paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "emmericp", "ovpn": "paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'oliveiradan', 'ovpn': 'oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "oliveiradan", "ovpn": "oliveiradan@opensuse-dev twDqMYwYsdYmbgyCpUnElw ec3ee80ddc747c3ca5e1455a122279f8e1e642c5c09aa9c2ca7fec142f55089e"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.675 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'mchangir', 'ovpn': 'mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mchangir", "ovpn": "mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.677 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'swagner', 'ovpn': 'swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "swagner", "ovpn": "swagner@ubuntu 64V1h0Se0FmBQNH7KLibbQ ad7c91e9e2f7f3999492d5e41fbbc993327d37929bd09606227367d75e5556ba"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'sjust', 'ovpn': 'sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sjust", "ovpn": "sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'yuvalif', 'ovpn': 'ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnVSKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "yuvalif", "ovpn": "ylifshit@localhost dyc2NU2pMz8NF8/uR2kMxA 3a6f1f9e55b5116f74d01ffbabdc339054088d257a16cf9fafcfe05b27fa678e\nyuvalif@medea gy0T/Hj/6DnV 2021-11-11T20:39:25.678 INFO:teuthology.task.ansible.out:SKyOKD0/ag 468fe695fad8610a8f2eba6d1015949e4e6dc3d62c50ecf03579928d45a8d2ae\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.687 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'ideepika', 'ovpn': 'deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ideepika", "ovpn": "deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.689 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'kkeithle', 'ovpn': 'kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kkeithle", "ovpn": "kkeithle@kkeithle.usersys.redhat.com FPnVevv1sp5hlWoJeDCe/g e5a1fa7ccf678b91ed570983d5420c98f109d507442c8e4dcd50803e0d71c852"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'gsalomon', 'ovpn': 'gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "gsalomon", "ovpn": "gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'emmericp', 'ovpn': 'paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "emmericp", "ovpn": "paul.emmerich@croit.io RN4hOorvA7irUg/3ViM9KQ 3bd06194186d2624cadf255fa1c38ddf7dded0a6d83dc6001cd55fcc0a899130"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.700 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'soumyakoduri', 'ovpn': 'skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "soumyakoduri", "ovpn": "skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.701 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'mchangir', 'ovpn': 'mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mchangir", "ovpn": "mchangir@indraprastha 1p71eDTdLMzkySFg0d5qEg bc4fda0ac9ef5cfbfab0cd9e5ed956c75b6280a04043c90c288e62b416b42da0"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'kyr', 'ovpn': 'kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kyr", "ovpn": "kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'sjust', 'ovpn': 'sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sjust", "ovpn": "sjust@pondermatic sCd6606QpID1PnHn0AxFag 46da7d4c77cb1238f83a34f99e774707110f997d88f3a0fd240aac9b7b7bbc85\nsjust@rex004 YS0in6YtQJHx5aUo7ZHi8Q bdd5977d05171a365539b19fae283ec2e3c7389516664692b9bbbaf98c7b61f4\nsjust@office w19UilyC/xu7uCzv0DnWRg ab20efc114b769bf4c2cf313eb30db09c2e2f8234992f120cfc3d1b8b347ed3c\nsam@deepthought 44sCi+GEfY0zjKo5M/4FiQ ed1eedd14ca68116a2000477fa078f8f736d0a15640723c32204bb30f14cb888\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.712 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'sseshasa', 'ovpn': 'sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sseshasa", "ovpn": "sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.714 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'ideepika', 'ovpn': 'deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ideepika", "ovpn": "deepika@x1carbon hyRj1/9UjWhVzhav5+LNKA e3f7d2882c2b1a410dbb5458c08f280814d652e34c87e96f8a2e8bef1928926f"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'rfriedma', 'ovpn': 'rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rfriedma", "ovpn": "rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.724 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'gsalomon', 'ovpn': 'gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "gsalomon", "ovpn": "gsalomon@home BrlCVxStNlEZGf5yYYAz1w 4b0c8d5b57dae1328c16d3017b59b632ccdfebe4135209fa97748c70ff00cc46"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.726 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'lmb', 'ovpn': 'lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "lmb", "ovpn": "lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'ksirivad', 'ovpn': 'ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ksirivad", "ovpn": "ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.737 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'soumyakoduri', 'ovpn': 'skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "soumyakoduri", "ovpn": "skoduri@localhost w/vFHwM/x4hUGlKJCteKiA 97933f044fdd8b3cf57ad744ce032728b06f354b0d719790def82b2f5b9a6133"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.739 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'ofriedma', 'ovpn': 'ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ofriedma", "ovpn": "ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.748 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'h4iresis', 'ovpn': 'h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "h4iresis", "ovpn": "h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.750 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'kyr', 'ovpn': 'kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kyr", "ovpn": "kyr@suse Xu4+Yi98Il4ETOAav6okqA 03bc46e5fac6346cd82ff681f756860f98e0c61168633ce23325efde11a1964a"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.759 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'shyukri', 'ovpn': 'shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "shyukri", "ovpn": "shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.762 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'scarvalhojr', 'ovpn': 'scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "scarvalhojr", "ovpn": "scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.771 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'sseshasa', 'ovpn': 'sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sseshasa", "ovpn": "sseshasa@thinkpad jyB1pr0I3qsDknkTnJTMjg 72ac1456e344c22fd940d0ba0e035aa3819ef7cd3891e53308aa92ba2dec8849"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.772 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'rakesh', 'ovpn': 'rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rakesh", "ovpn": "rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.772 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'ryg-', 'ovpn': 'ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ryg-", "ovpn": "ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.783 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'rfriedma', 'ovpn': 'rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rfriedma", "ovpn": "rfriedma@rflap.redhat.com 5+OUPoyz8K0M0kcymdQOjA 40ce705001f31d7156c965228938cd4b02ae1a2c43dac1bbcd1b538e70312189"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.785 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "prsrivas", "ovpn": "prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'lmb', 'ovpn': 'lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "lmb", "ovpn": "lmb@hermes LMS8kAikL0iqw2S6IbXa3Q f57a493b31e7ed02a2563dd4295278d4842dc698b4c635d011a8d2b4b1fd5c2b"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.795 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'zdover', 'ovpn': 'zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "zdover", "ovpn": "zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.796 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'ksirivad', 'ovpn': 'ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ksirivad", "ovpn": "ksirivad@ksirivad prZzE08FqnY6hFtGk0Q6XQ 2ef0878b0050cf28775813fe0991f9a746c07e61920280ce29ee69088eda5efc\nksirivad@macbook 0ivvJOZVAywT15aJCDF2xw 0066940a99b2d37dcd856c8622f48e0d8cb59c092f8f96cc2352ec0bc979ce45\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'rraja', 'ovpn': 'rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rraja", "ovpn": "rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.806 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'ofriedma', 'ovpn': 'ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ofriedma", "ovpn": "ofriedma@myhost IIifKuFYfX5EqO3mhV8lAg 82cb23a98361f3624b2f1f4e0f694f9439ae7735b171ba5e3bb5f0744f5ddfc4"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.808 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'toabctl', 'ovpn': 'toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "toabctl", "ovpn": "toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'h4iresis', 'ovpn': 'h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "h4iresis", "ovpn": "h4iresis@fedora NyMdx+dAfmujNSDB63kI+g bcf6c36594585e0fbf9f037bf1a13fe63db4b82a52ff1349349fac748b5c4e52"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.818 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'mgfritch', 'ovpn': 'mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mgfritch", "ovpn": "mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.820 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'shyukri', 'ovpn': 'shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "shyukri", "ovpn": "shyukri@t480 KEbvWFIOWqkHsLgffi6GMQ fcd405d0cf6b2c7ba53397c4ed1156e852317fb02bcd6f0e98e5eb3c1c3aa4b5"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'xiubli', 'ovpn': 'root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "xiubli", "ovpn": "root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'scarvalhojr', 'ovpn': 'scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "scarvalhojr", "ovpn": "scarvalhojr@milestones utvFbB+X1G75M589jLfFXQ 9c62bf416c9204b25028ba0e5bd45f754c4aa8f84a85b9d5ded02aa83f03e82e"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.831 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'jafaj', 'ovpn': 'jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jafaj", "ovpn": "jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.832 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'rakesh', 'ovpn': 'rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rakesh", "ovpn": "rakesh@fedora CXsWP7Wzu2C+bTN6mIeMiw 2f4f492cd085f1f46d1d5018aeb7705ba855c4481e230a8188b8f5ea9994b91b"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'jschmid', 'ovpn': 'jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jschmid", "ovpn": "jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'ryg-', 'ovpn': 'ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ryg-", "ovpn": "ryg@wombat Y8asVyh6Qu+EEmWlRd1zXA d4923cf7be8e59728ea290aaec7670903b39b6444e5d303016338ddf5ab7837f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.843 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'kachau', 'ovpn': 'kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kachau", "ovpn": "kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.845 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'prsrivas', 'ovpn': 'prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "prsrivas", "ovpn": "prsrivas@t480 dvGwTZM2FivQPrMGmkY53g 80afabbb654af66ec01463d0e0425be5643eaca10bbf756f2b8313645d2b8da3"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'khiremat', 'ovpn': 'khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "khiremat", "ovpn": "khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.855 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'zdover', 'ovpn': 'zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "zdover", "ovpn": "zdover@lenovot5301 88ArNhsAMXq/nkSC9tKDig 131b8214bbea52345220522f8dd13c768bc5c1e41e4d164e7138719cfc3295d3"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.856 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'kapandya', 'ovpn': 'kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kapandya", "ovpn": "kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'rraja', 'ovpn': 'rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "rraja", "ovpn": "rraja@p50 k0lrZNIKWMH/VrZupJ9diA 03bfe34ee6e9a541e10426a3cc1e1046655420d4d0a79ce5667bec1b469a0100"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.866 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'prasad', 'ovpn': 'prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "prasad", "ovpn": "prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.868 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'toabctl', 'ovpn': 'toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "toabctl", "ovpn": "toabctl@minze a3I/7JakMEEANsNXCYM5XA b9715003902937328801993a359da43c61ab0be8d17f92371a48dca0821fb13f"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'vikhyat', 'ovpn': 'vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vikhyat", "ovpn": "vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'mgfritch', 'ovpn': 'mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mgfritch", "ovpn": "mgfritch@laptop 94prpsaAkBwTzMmF9JR4WA 3adb8aad97b1f78c74d3ae91f21f4b165749843dacabad74f0039212c5a5db39"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.879 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'shraddhaag', 'ovpn': 'shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "shraddhaag", "ovpn": "shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.881 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'xiubli', 'ovpn': 'root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "xiubli", "ovpn": "root@lxbceph0 /BNUvfmLPqzfbIz/Jzlqkw 3e109dcb826d45a6ee7e8195fc025b257b3d369e135400bf0a4bf503d48503e3\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'gkyratsas', 'ovpn': 'gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "gkyratsas", "ovpn": "gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'jafaj', 'ovpn': 'jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jafaj", "ovpn": "jan@jfsuselaptop ddAKCkWSc1rEhQadU15jkA 90ddc588ca94ab776b539a1c5246a216dfbebabba3709ae803d386f978b65c6f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.891 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'prateek', 'ovpn': 'prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "prateek", "ovpn": "prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.893 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'jschmid', 'ovpn': 'jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jschmid", "ovpn": "jschmid@zulu BqyepNoWzZbbsslJbvCuYg 4ce9b431fdd8a06856cfe00b88665da11dde1cacd72d40bf3dbe54ff633066ef"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'jelopez', 'ovpn': 'jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jelopez", "ovpn": "jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.903 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'kachau', 'ovpn': 'kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kachau", "ovpn": "kachau@thinkpad dX7+Zp6Z+rMSZTbxL52c7g 59d0d7be858e4a3769eaca4563399989298004a271f60b08cc36bf4130e089fe"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.905 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'dcain', 'ovpn': 'dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dcain", "ovpn": "dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'khiremat', 'ovpn': 'khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "khiremat", "ovpn": "khiremat@kotresh-T490s 7o9JjenDm1h34Zmd2SWO2g 791981cd84bd4c80bf3ff6076ebc64b3ebf8432903e36d1a49bbee27561f8315"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.915 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'sunny', 'ovpn': 'sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sunny", "ovpn": "sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.917 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'kapandya', 'ovpn': 'kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kapandya", "ovpn": "kapandya@thinkpad g4JoonWShBhyS6fEdaHh7A 2990a987bf7afee6e3c8b5236b3dac813a57dbdce47ec63965c40b1b7bdafd67"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'xuxuehan', 'ovpn': 'xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "xuxuehan", "ovpn": "xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'prasad', 'ovpn': 'prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "prasad", "ovpn": "prasad@macprasad tJYuW/3nDX8ikaLyJQdpAQ 1544f87809a170bb61069b588630467b05283a1347945b9545205b0a28110f89"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.927 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'aclewett', 'ovpn': 'aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "aclewett", "ovpn": "aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.929 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'vikhyat', 'ovpn': 'vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vikhyat", "ovpn": "vikhyat@redhat ae77Y7dK1BxLEfQvAOJo8A a62d65a2aa09a75dca299ce0424caf6bcbc52696b22aa1567432aac34d78f18f"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'cblum', 'ovpn': 'cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "cblum", "ovpn": "cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'shraddhaag', 'ovpn': 'shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "shraddhaag", "ovpn": "shraddhaag@asus w1lmx5nwk8QQ/4u3UfGsJQ 98c5840f994a5e35b234375b07b1782f9a92bdcd5cf73e5847027045ab88e6d2"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.940 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'mlatimer', 'ovpn': 'mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mlatimer", "ovpn": "mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.941 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'gkyratsas', 'ovpn': 'gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "gkyratsas", "ovpn": "gkyratsas@suse.com PO5Ut+2Z4kCMjaumOyA+1A 70fafe30ccb801c8c0a059a7eb3a842a2d08015b828c63bc1a702caf056014c7"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'shyamsundarr', 'ovpn': 'shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "shyamsundarr", "ovpn": "shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'prateek', 'ovpn': 'prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "prateek", "ovpn": "prateek@linux kj4pU4yLf+76XxY0mvoIwg c24292e824e3e43ebf7465f1079449ce80bf26311426ae3914cdf1fe8d591c67"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.952 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'shuzhenyi', 'ovpn': 'bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "shuzhenyi", "ovpn": "bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.954 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'jelopez', 'ovpn': 'jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jelopez", "ovpn": "jelopez@imac XbzY2mjr57lSPExJp0ZBgA 4f5dfad692c25293cd58393bd8ddae91b4dc0b2561c860072e0665933531208e"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'mdipalma', 'ovpn': 'mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mdipalma", "ovpn": "mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.964 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'dcain', 'ovpn': 'dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dcain", "ovpn": "dcain@fullsteam 7vptxC1ZWCXLUV3FJwWQAg 2d73728e16c5f724e5f28eb326c24b8e13e7b361bcbaf506809f539e80223389"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.964 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'kchang', 'ovpn': 'kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kchang", "ovpn": "kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.966 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'sunny', 'ovpn': 'sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sunny", "ovpn": "sunny@devBox 81r/AKNOWhvaD0ESYy6tTg 58b86902f858bc3f1edb12cfc41ce10ed163985db2f7c253de08b7f1c310fe11"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'vereddy', 'ovpn': 'vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vereddy", "ovpn": "vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.966 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'xuxuehan', 'ovpn': 'xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "xuxuehan", "ovpn": "xuxuehan@xuxuehan-PC aJDFkJ1+XzbovMyvRiFCAQ 5a71908d9bc8372ce0fff3bdbce4ea0e27eb11af0ed30e573f93d7a0b4335279\nxuxuehan@xuxuehan-notebook LzP6obPSYuao0iCJLQ/92A 3af116b690da581c77912a0c5fe184c84ca78526f67331db15714bc4ffddbcdc\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.977 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'benhanokh', 'ovpn': 'gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "benhanokh", "ovpn": "gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.978 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'aclewett', 'ovpn': 'aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "aclewett", "ovpn": "aclewett@mac bP0KgOkhW1GxTiugD3jMQA 90009b65da8175e083ee0e79d4953268f44493b98c2b532854da8d9e1d14e798"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.988 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'jsalomon', 'ovpn': 'jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jsalomon", "ovpn": "jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:25.990 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'cblum', 'ovpn': 'cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "cblum", "ovpn": "cblum@ ad4mZ9Cg0u6H8AdARvQUZw 4e3cec570cbe42786e5190de4194c2937f1073aef62a25b61bd85e05ee23db56"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'jolmomar', 'ovpn': 'jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jolmomar", "ovpn": "jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'mlatimer', 'ovpn': 'mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mlatimer", "ovpn": "mlatimer@lx5510 3lc4nP664Dmg63yr8yKG7w 3778fb16fb62d1a7d61f41cf2c2e72fd9acaae2805de2dc34497ef41ee1ad64f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.000 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'julpark', 'ovpn': 'julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "julpark", "ovpn": "julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.002 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'shyamsundarr', 'ovpn': 'shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "shyamsundarr", "ovpn": "shyamsundarr@bartimaeus HUEzMC00f5f/tdydddWGNw e9032244bda086ecca6abc4f525296f9a125e4bbdb768d8c76c6d53d02f475d5"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'pdhange', 'ovpn': 'pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pdhange", "ovpn": "pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'shuzhenyi', 'ovpn': 'bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "shuzhenyi", "ovpn": "bigwhite@wrjj37deMacBook-Pro-2.local VoTYmzsOPazeWcH+DOHXKw d7502e96b37ebc2ee5e0c14c6026f10e2e72b49be70dd1c5c7e9751fafcd8501"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.013 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'dpivonka', 'ovpn': 'dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dpivonka", "ovpn": "dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.014 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'mdipalma', 'ovpn': 'mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mdipalma", "ovpn": "mdipalma@mdmac IVn9Oy/SxZEmDgSWSeo3jA ea4dc08cb39ba55ade357c4949bf3f7d4d0322171ff5b5dac3c06d47cdc38f14"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.024 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'kchang', 'ovpn': 'kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kchang", "ovpn": "kchang@kiefer-uchi Vy8tg3hSizdX5fIAJFgG/A 912a49a77b5722fdf4b9e304ee329c7b808728d73bb1e1f70cf55bd1221a3da3"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.025 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'adking', 'ovpn': 'adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "adking", "ovpn": "adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'vereddy', 'ovpn': 'vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "vereddy", "ovpn": "vereddy@vereddy.remote.csb 8jHiMDBfgQxu/8+MTlPR1A 07898add1a48c50647de4a81ad23db0c8545202cb56d3172db9b22df32c6b4a2"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'amathuri', 'ovpn': 'amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "amathuri", "ovpn": "amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.036 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'benhanokh', 'ovpn': 'gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "benhanokh", "ovpn": "gbenhano@localhost.localdomain uMf63iCCcqDt1rUnOSujaw 8c554c50bf8e27d3373515168e59f4ed3e4c48bace5689838268569948545ed5"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.038 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'nizamial09', 'ovpn': 'nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "nizamial09", "ovpn": "nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'jsalomon', 'ovpn': 'jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jsalomon", "ovpn": "jsalomon@Josh-laptop 9V9NVwrt57LxXiqRh5Q7PQ 4cf0b0dfa39cf1e72f51bbfed19b4d88bdb286d09e54a99eb35721e33b1ae7b0"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'dachary', 'ovpn': 'loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dachary", "ovpn": "loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.048 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'jolmomar', 'ovpn': 'jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jolmomar", "ovpn": "jolmomar@thinkpad CnoIzb/786lZh1NINj0xAA e60eeb560654da4b84977a4f9dbccd599bf9460980985ee5726595eb4a46e640"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.049 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'amctagga', 'ovpn': 'amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "amctagga", "ovpn": "amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.049 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'julpark', 'ovpn': 'julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "julpark", "ovpn": "julpark@thinkpad svj+nOCQG95zuBUwdJoluQ 7252f05ef02b16fd74d931d6cf937304afbb098dbdbde13d377e884fe35e2fb8"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.060 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'kevinz', 'ovpn': 'kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kevinz", "ovpn": "kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.060 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'pdhange', 'ovpn': 'pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pdhange", "ovpn": "pdhange@mauli hqKEk1OuI2Wmi1CAcJk5CQ f49d854572b5efe9514b4d741e336adb3534d00b22b0a505142d82eccc2f7535"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.061 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'liu-chunmei', 'ovpn': 'liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "liu-chunmei", "ovpn": "liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.061 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'dpivonka', 'ovpn': 'dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dpivonka", "ovpn": "dpivonka@thinkpad 6vBnL22Dfp8d9grCNtSoHw c8270453301dbec76f8070809078087ff892c74360b80a82002d58c50af19ec4"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.072 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'cyx1231st', 'ovpn': 'cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "cyx1231st", "ovpn": "cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.074 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'adking', 'ovpn': 'adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "adking", "ovpn": "adking@laptop1 VkjYUD1hJpR8sFhrH6MIcg 28c610bc9f7321de315923288d24650bdb4bc90593b423f1f59db16d67b6d625"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'sunnagar', 'ovpn': 'sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sunnagar", "ovpn": "sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.084 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'amathuri', 'ovpn': 'amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "amathuri", "ovpn": "amathuri@thinkpad WDHZL4VspBmlz5dzRlAGUQ 7f2c5d8e2554b869a09fdc20d6e4aabffa8200ef8e2b8d1bc951bfc673757267\namathuri@lenovothinkpad MhlrEG4X+iKq2d2GxwMbmA 812cfea500340140144ecb27f7851b86d8daff658f173189fceec8e61b59cbf2\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.085 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'lflores', 'ovpn': 'lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "lflores", "ovpn": "lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'nizamial09', 'ovpn': 'nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "nizamial09", "ovpn": "nia@redhat OA+N80Ps0Ua7PfuVF4+M8w a9b0ecf7a28ef91e0653b0193f3f95967d6a671eff21ee8a99620a5a972ba409"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'jsawaya', 'ovpn': 'jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jsawaya", "ovpn": "jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.096 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'dachary', 'ovpn': 'loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "dachary", "ovpn": "loic@tulipe 1NGxatlPDW6m9FLwzG2Gng cc19cfe2d2101909e9e0ae6b207d7e8dccafd8ec1f4594bdd006dda6b67a80b5"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.098 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'markw8973240', 'ovpn': 'shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "markw8973240", "ovpn": "shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'amctagga', 'ovpn': 'amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "amctagga", "ovpn": "amctagga@redhat20 DsFh00Ttjtomgs5nlTX3SA 713280e601e14a8fecfe190490e362f5f97bba3517b2bcfd4abd5230a45fbaa1"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.108 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'luisrico', 'ovpn': 'lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "luisrico", "ovpn": "lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.110 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'kevinz', 'ovpn': 'kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "kevinz", "ovpn": "kevinz@linaroceph a2GchLhkaayg78xuUnI0sA b545e84634e614099b00d50e593933eabf0793035eaa439b6d82ec669f69de7f"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'matan', 'ovpn': 'matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "matan", "ovpn": "matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'liu-chunmei', 'ovpn': 'liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "liu-chunmei", "ovpn": "liu-chunmei@otccldstore gKMLlCSH99LCDse00ja0Kg 2c140368bca0e48102330631f91e5299298461a24cab483b9765293eafb2050c"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.120 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'ozeneva', 'ovpn': 'ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ozeneva", "ovpn": "ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.122 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'cyx1231st', 'ovpn': 'cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "cyx1231st", "ovpn": "cyx1231st@yingxin-server C3qUn5JXYFehgyfol2lnhw c5f306797eac838797ea99b07bd0f5e8e185052887b82ba5c343a42eb271747a"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'sudhanaruna', 'ovpn': 'ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sudhanaruna", "ovpn": "ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.132 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'sunnagar', 'ovpn': 'sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sunnagar", "ovpn": "sunil@sepia-lab j6h8NzyTGEE6WHYT3uiOXA eda7763cdd544d8e7198cefee77f38641d44c8a6aa4d37b8b1429620641809ae"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.134 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'zl31wang', 'ovpn': 'zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "zl31wang", "ovpn": "zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'lflores', 'ovpn': 'lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "lflores", "ovpn": "lflores@fedora CqF8+5Scni+GHCfHkm3AAA 0771c83aa58141af4704f4b01bba4490638ee2033fdabe5a2a4b8e06768b29c2"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'mayagilad', 'ovpn': 'mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mayagilad", "ovpn": "mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.144 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'jsawaya', 'ovpn': 'jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "jsawaya", "ovpn": "jsawaya@thinkpad jEJaJNljo4e2sHstFCNljQ d0042e4c40344f7fa350d3e0b748686aea00c4757ffbe32a41f3b119463d9f46"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.146 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'arjsharm', 'ovpn': 'arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "arjsharm", "ovpn": "arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'markw8973240', 'ovpn': 'shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "markw8973240", "ovpn": "shuai@mac d7Tqhwnt5D7+jeeqKQ375w e6ac2fb0750350a007d00bba1354989402199dc012150a84805be57a92e6f06d"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'melissali', 'ovpn': 'melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "melissali", "ovpn": "melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.156 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'luisrico', 'ovpn': 'lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "luisrico", "ovpn": "lricomor@lricomor-mac DI4SNq2EznedeK8Cztt2ug e05bc16840e8c3c45f641852101ad921abc248013ec89a599c27c275767b5347"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.158 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'ionutbalutoiu', 'ovpn': 'ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ionutbalutoiu", "ovpn": "ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'matan', 'ovpn': 'matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "matan", "ovpn": "matan@main-win vEAxj3ZM9DjxBi/jfjc+jQ 9c30b8fcef9bdc804090947a92e1e60cc21d3f280f261bc3f4cecac64541dc6e\nmatan@thinkpad JJuwaWbac2lbcH5pCGON1w 521fe30e3a950915a1b538913c1af76198c4133018b7869d348740c708b38a7d\n"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.168 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'pdiazbou', 'ovpn': 'pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pdiazbou", "ovpn": "pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.170 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'ozeneva', 'ovpn': 'ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ozeneva", "ovpn": "ozeneva@omri qB3om9JUxgVneOibFeRq6Q 92277afcd0bce888f60dceab4b2e8695ebbf7241dba09e3350a12c4fffd6ebd5"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'xuwang', 'ovpn': 'xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "xuwang", "ovpn": "xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.180 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'sudhanaruna', 'ovpn': 'ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "sudhanaruna", "ovpn": "ArunaSudhan@sudhan-linux t/fhf5WR4vQxW3F/Lx7Atw 09248b17c9abfe802ca4be1b93ad318138f54fb60b073ff857c2effadb27a997"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.181 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'blaine', 'ovpn': 'blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "blaine", "ovpn": "blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'zl31wang', 'ovpn': 'zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "zl31wang", "ovpn": "zl31wang@sjtu Et6K7OVsdRTnvOY5iuvUxA a7f66c06b405df269ac8f28a4b2c9f00d087091082c53ed5e03517e3811d9fc2"}, "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item={'name': 'choffman', 'ovpn': 'choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "choffman", "ovpn": "choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.211 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'mayagilad', 'ovpn': 'mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "mayagilad", "ovpn": "mayagilad@mayagilad-ThinkPad-T480 AGw0uOFkRU6SX8Kp3nyGuA d29a792aad9cc6038dfc2e3471f6556b50ef73562b1bc6231c39d8daf20a19d0"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.212 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'arjsharm', 'ovpn': 'arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "arjsharm", "ovpn": "arjun@thinkpad_local 06szrDO7vKFS7BSicSbDsw e1284059b3e0604191a0f35e2841f22835b0e75249e637d8297edbee458ccf39"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'melissali', 'ovpn': 'melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "melissali", "ovpn": "melissali@melissali-ubuntu pooAecOHrOtZ3gXewVMxnw 36860a4078c773f47e1a58d2f120ec79e468142120ff6a816cb6ffbb9fd9ef07"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'ionutbalutoiu', 'ovpn': 'ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "ionutbalutoiu", "ovpn": "ionutbalutoiu@macbook ruUdfqkSfEhKctGGLeQvkw 0a6cab2332041d56ca8b1216242195af2ad8d0ec6360fd6c1dd080f5f43240e1"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.233 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:26 +0000 (0:00:01.827) 0:01:46.489 ***** 2021-11-11T20:39:26.242 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'pdiazbou', 'ovpn': 'pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "pdiazbou", "ovpn": "pdiazbou@thinkpad BNtoPcISV462wNeprZDOcg f6faca031e6549014132b4c66e53721c9d934a09d0d9d0f36e2dd31f90be1ce4"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.242 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'xuwang', 'ovpn': 'xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "xuwang", "ovpn": "xuwang@laptop 2U0uRjvVh8eE4iSkFzK2Kg 2a9db1c375a032cc0a892ca2d0721f9d7303e2db5a7eab647870d7c2022ee2e8"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.244 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'blaine', 'ovpn': 'blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "blaine", "ovpn": "blaine@laptop RkDSJBkoRKI0UaxdQuMNCw abcfdade65dd772c066c0c63e712fb03240d676d72eb341186b0c0a94d4a4547"}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'name': 'choffman', 'ovpn': 'choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c'}) => {"ansible_loop_var": "item", "changed": false, "item": {"name": "choffman", "ovpn": "choffman@choffman.net 56rg/lHPXPRyVivND7+0Uw ca591258cda719a5a0bf5d5f4fa87976773625249d29ff6012ad6efeac33da2c"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.296 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:26 +0000 (0:00:00.062) 0:01:46.552 ***** 2021-11-11T20:39:26.301 INFO:teuthology.task.ansible.out: TASK [Filter the revoked_users list] ******************************************* 2021-11-11T20:39:26.303 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.326 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:26 +0000 (0:00:00.030) 0:01:46.582 ***** 2021-11-11T20:39:26.364 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:26.387 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:26 +0000 (0:00:00.060) 0:01:46.643 ***** 2021-11-11T20:39:26.600 INFO:teuthology.task.ansible.out: TASK [Remove revoked users] **************************************************** 2021-11-11T20:39:26.602 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=zcerza) => {"ansible_loop_var": "item", "changed": false, "item": "zcerza", "name": "zcerza", "state": "absent"} 2021-11-11T20:39:26.664 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=zcerza) => {"ansible_loop_var": "item", "changed": false, "item": "zcerza", "name": "zcerza", "state": "absent"} 2021-11-11T20:39:26.790 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=aschoen) => {"ansible_loop_var": "item", "changed": false, "item": "aschoen", "name": "aschoen", "state": "absent"} 2021-11-11T20:39:26.853 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=aschoen) => {"ansible_loop_var": "item", "changed": false, "item": "aschoen", "name": "aschoen", "state": "absent"} 2021-11-11T20:39:26.987 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=andrew) => {"ansible_loop_var": "item", "changed": false, "item": "andrew", "name": "andrew", "state": "absent"} 2021-11-11T20:39:27.051 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=andrew) => {"ansible_loop_var": "item", "changed": false, "item": "andrew", "name": "andrew", "state": "absent"} 2021-11-11T20:39:27.177 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=sweil) => {"ansible_loop_var": "item", "changed": false, "item": "sweil", "name": "sweil", "state": "absent"} 2021-11-11T20:39:27.251 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=sweil) => {"ansible_loop_var": "item", "changed": false, "item": "sweil", "name": "sweil", "state": "absent"} 2021-11-11T20:39:27.367 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=brad) => {"ansible_loop_var": "item", "changed": false, "item": "brad", "name": "brad", "state": "absent"} 2021-11-11T20:39:27.430 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=brad) => {"ansible_loop_var": "item", "changed": false, "item": "brad", "name": "brad", "state": "absent"} 2021-11-11T20:39:27.556 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=kefu) => {"ansible_loop_var": "item", "changed": false, "item": "kefu", "name": "kefu", "state": "absent"} 2021-11-11T20:39:27.630 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=kefu) => {"ansible_loop_var": "item", "changed": false, "item": "kefu", "name": "kefu", "state": "absent"} 2021-11-11T20:39:27.744 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=shylesh) => {"ansible_loop_var": "item", "changed": false, "item": "shylesh", "name": "shylesh", "state": "absent"} 2021-11-11T20:39:27.817 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=shylesh) => {"ansible_loop_var": "item", "changed": false, "item": "shylesh", "name": "shylesh", "state": "absent"} 2021-11-11T20:39:27.932 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=gmeno) => {"ansible_loop_var": "item", "changed": false, "item": "gmeno", "name": "gmeno", "state": "absent"} 2021-11-11T20:39:28.005 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=gmeno) => {"ansible_loop_var": "item", "changed": false, "item": "gmeno", "name": "gmeno", "state": "absent"} 2021-11-11T20:39:28.132 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=alfredodeza) => {"ansible_loop_var": "item", "changed": false, "item": "alfredodeza", "name": "alfredodeza", "state": "absent"} 2021-11-11T20:39:28.195 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=alfredodeza) => {"ansible_loop_var": "item", "changed": false, "item": "alfredodeza", "name": "alfredodeza", "state": "absent"} 2021-11-11T20:39:28.320 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=trhoden) => {"ansible_loop_var": "item", "changed": false, "item": "trhoden", "name": "trhoden", "state": "absent"} 2021-11-11T20:39:28.393 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=trhoden) => {"ansible_loop_var": "item", "changed": false, "item": "trhoden", "name": "trhoden", "state": "absent"} 2021-11-11T20:39:28.508 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=nishtha) => {"ansible_loop_var": "item", "changed": false, "item": "nishtha", "name": "nishtha", "state": "absent"} 2021-11-11T20:39:28.582 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=nishtha) => {"ansible_loop_var": "item", "changed": false, "item": "nishtha", "name": "nishtha", "state": "absent"} 2021-11-11T20:39:28.698 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=yguang) => {"ansible_loop_var": "item", "changed": false, "item": "yguang", "name": "yguang", "state": "absent"} 2021-11-11T20:39:28.771 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=yguang) => {"ansible_loop_var": "item", "changed": false, "item": "yguang", "name": "yguang", "state": "absent"} 2021-11-11T20:39:28.898 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=sdieffen) => {"ansible_loop_var": "item", "changed": false, "item": "sdieffen", "name": "sdieffen", "state": "absent"} 2021-11-11T20:39:28.962 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=sdieffen) => {"ansible_loop_var": "item", "changed": false, "item": "sdieffen", "name": "sdieffen", "state": "absent"} 2021-11-11T20:39:29.088 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=brian) => {"ansible_loop_var": "item", "changed": false, "item": "brian", "name": "brian", "state": "absent"} 2021-11-11T20:39:29.162 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=brian) => {"ansible_loop_var": "item", "changed": false, "item": "brian", "name": "brian", "state": "absent"} 2021-11-11T20:39:29.278 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=pmcgarry) => {"ansible_loop_var": "item", "changed": false, "item": "pmcgarry", "name": "pmcgarry", "state": "absent"} 2021-11-11T20:39:29.352 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=pmcgarry) => {"ansible_loop_var": "item", "changed": false, "item": "pmcgarry", "name": "pmcgarry", "state": "absent"} 2021-11-11T20:39:29.467 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=karnan) => {"ansible_loop_var": "item", "changed": false, "item": "karnan", "name": "karnan", "state": "absent"} 2021-11-11T20:39:29.540 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=karnan) => {"ansible_loop_var": "item", "changed": false, "item": "karnan", "name": "karnan", "state": "absent"} 2021-11-11T20:39:29.657 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=ryneli) => {"ansible_loop_var": "item", "changed": false, "item": "ryneli", "name": "ryneli", "state": "absent"} 2021-11-11T20:39:29.731 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=ryneli) => {"ansible_loop_var": "item", "changed": false, "item": "ryneli", "name": "ryneli", "state": "absent"} 2021-11-11T20:39:29.847 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=dlambrig) => {"ansible_loop_var": "item", "changed": false, "item": "dlambrig", "name": "dlambrig", "state": "absent"} 2021-11-11T20:39:29.922 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=dlambrig) => {"ansible_loop_var": "item", "changed": false, "item": "dlambrig", "name": "dlambrig", "state": "absent"} 2021-11-11T20:39:30.038 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=icolle) => {"ansible_loop_var": "item", "changed": false, "item": "icolle", "name": "icolle", "state": "absent"} 2021-11-11T20:39:30.112 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=icolle) => {"ansible_loop_var": "item", "changed": false, "item": "icolle", "name": "icolle", "state": "absent"} 2021-11-11T20:39:30.228 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=soumya) => {"ansible_loop_var": "item", "changed": false, "item": "soumya", "name": "soumya", "state": "absent"} 2021-11-11T20:39:30.302 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=soumya) => {"ansible_loop_var": "item", "changed": false, "item": "soumya", "name": "soumya", "state": "absent"} 2021-11-11T20:39:30.427 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=jspray) => {"ansible_loop_var": "item", "changed": false, "item": "jspray", "name": "jspray", "state": "absent"} 2021-11-11T20:39:30.491 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=jspray) => {"ansible_loop_var": "item", "changed": false, "item": "jspray", "name": "jspray", "state": "absent"} 2021-11-11T20:39:30.617 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=erwan) => {"ansible_loop_var": "item", "changed": false, "item": "erwan", "name": "erwan", "state": "absent"} 2021-11-11T20:39:30.681 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=erwan) => {"ansible_loop_var": "item", "changed": false, "item": "erwan", "name": "erwan", "state": "absent"} 2021-11-11T20:39:30.811 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=jj) => {"ansible_loop_var": "item", "changed": false, "item": "jj", "name": "jj", "state": "absent"} 2021-11-11T20:39:30.885 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=jj) => {"ansible_loop_var": "item", "changed": false, "item": "jj", "name": "jj", "state": "absent"} 2021-11-11T20:39:30.999 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=amarangone) => {"ansible_loop_var": "item", "changed": false, "item": "amarangone", "name": "amarangone", "state": "absent"} 2021-11-11T20:39:31.072 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=amarangone) => {"ansible_loop_var": "item", "changed": false, "item": "amarangone", "name": "amarangone", "state": "absent"} 2021-11-11T20:39:31.187 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=oprypin) => {"ansible_loop_var": "item", "changed": false, "item": "oprypin", "name": "oprypin", "state": "absent"} 2021-11-11T20:39:31.261 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=oprypin) => {"ansible_loop_var": "item", "changed": false, "item": "oprypin", "name": "oprypin", "state": "absent"} 2021-11-11T20:39:31.376 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=adamyanova) => {"ansible_loop_var": "item", "changed": false, "item": "adamyanova", "name": "adamyanova", "state": "absent"} 2021-11-11T20:39:31.449 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=adamyanova) => {"ansible_loop_var": "item", "changed": false, "item": "adamyanova", "name": "adamyanova", "state": "absent"} 2021-11-11T20:39:31.574 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=sbillah) => {"ansible_loop_var": "item", "changed": false, "item": "sbillah", "name": "sbillah", "state": "absent"} 2021-11-11T20:39:31.647 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=sbillah) => {"ansible_loop_var": "item", "changed": false, "item": "sbillah", "name": "sbillah", "state": "absent"} 2021-11-11T20:39:31.762 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=onyb) => {"ansible_loop_var": "item", "changed": false, "item": "onyb", "name": "onyb", "state": "absent"} 2021-11-11T20:39:31.836 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=onyb) => {"ansible_loop_var": "item", "changed": false, "item": "onyb", "name": "onyb", "state": "absent"} 2021-11-11T20:39:31.951 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=jwilliamson) => {"ansible_loop_var": "item", "changed": false, "item": "jwilliamson", "name": "jwilliamson", "state": "absent"} 2021-11-11T20:39:32.024 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=jwilliamson) => {"ansible_loop_var": "item", "changed": false, "item": "jwilliamson", "name": "jwilliamson", "state": "absent"} 2021-11-11T20:39:32.139 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=kmroz) => {"ansible_loop_var": "item", "changed": false, "item": "kmroz", "name": "kmroz", "state": "absent"} 2021-11-11T20:39:32.212 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=kmroz) => {"ansible_loop_var": "item", "changed": false, "item": "kmroz", "name": "kmroz", "state": "absent"} 2021-11-11T20:39:32.327 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=shehbazj) => {"ansible_loop_var": "item", "changed": false, "item": "shehbazj", "name": "shehbazj", "state": "absent"} 2021-11-11T20:39:32.411 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=shehbazj) => {"ansible_loop_var": "item", "changed": false, "item": "shehbazj", "name": "shehbazj", "state": "absent"} 2021-11-11T20:39:32.515 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=abhishekvrshny) => {"ansible_loop_var": "item", "changed": false, "item": "abhishekvrshny", "name": "abhishekvrshny", "state": "absent"} 2021-11-11T20:39:32.599 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=abhishekvrshny) => {"ansible_loop_var": "item", "changed": false, "item": "abhishekvrshny", "name": "abhishekvrshny", "state": "absent"} 2021-11-11T20:39:32.713 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=asheplyakov) => {"ansible_loop_var": "item", "changed": false, "item": "asheplyakov", "name": "asheplyakov", "state": "absent"} 2021-11-11T20:39:32.786 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=asheplyakov) => {"ansible_loop_var": "item", "changed": false, "item": "asheplyakov", "name": "asheplyakov", "state": "absent"} 2021-11-11T20:39:32.900 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=liupan) => {"ansible_loop_var": "item", "changed": false, "item": "liupan", "name": "liupan", "state": "absent"} 2021-11-11T20:39:32.984 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=liupan) => {"ansible_loop_var": "item", "changed": false, "item": "liupan", "name": "liupan", "state": "absent"} 2021-11-11T20:39:33.088 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=adeza) => {"ansible_loop_var": "item", "changed": false, "item": "adeza", "name": "adeza", "state": "absent"} 2021-11-11T20:39:33.171 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=adeza) => {"ansible_loop_var": "item", "changed": false, "item": "adeza", "name": "adeza", "state": "absent"} 2021-11-11T20:39:33.286 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=pranith) => {"ansible_loop_var": "item", "changed": false, "item": "pranith", "name": "pranith", "state": "absent"} 2021-11-11T20:39:33.359 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=pranith) => {"ansible_loop_var": "item", "changed": false, "item": "pranith", "name": "pranith", "state": "absent"} 2021-11-11T20:39:33.473 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=dorinda) => {"ansible_loop_var": "item", "changed": false, "item": "dorinda", "name": "dorinda", "state": "absent"} 2021-11-11T20:39:33.556 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=dorinda) => {"ansible_loop_var": "item", "changed": false, "item": "dorinda", "name": "dorinda", "state": "absent"} 2021-11-11T20:39:33.660 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=zyan) => {"ansible_loop_var": "item", "changed": false, "item": "zyan", "name": "zyan", "state": "absent"} 2021-11-11T20:39:33.754 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=zyan) => {"ansible_loop_var": "item", "changed": false, "item": "zyan", "name": "zyan", "state": "absent"} 2021-11-11T20:39:33.858 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=jdillaman) => {"ansible_loop_var": "item", "changed": false, "item": "jdillaman", "name": "jdillaman", "state": "absent"} 2021-11-11T20:39:33.941 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=jdillaman) => {"ansible_loop_var": "item", "changed": false, "item": "jdillaman", "name": "jdillaman", "state": "absent"} 2021-11-11T20:39:35.534 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item=davidz) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "davidz", "name": "davidz", "remove": false, "state": "absent"} 2021-11-11T20:39:35.618 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item=davidz) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "davidz", "name": "davidz", "remove": false, "state": "absent"} 2021-11-11T20:39:37.064 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item=wusui) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "wusui", "name": "wusui", "remove": false, "state": "absent"} 2021-11-11T20:39:37.220 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item=wusui) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "wusui", "name": "wusui", "remove": false, "state": "absent"} 2021-11-11T20:39:38.576 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item=nwatkins) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "nwatkins", "name": "nwatkins", "remove": false, "state": "absent"} 2021-11-11T20:39:38.764 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item=nwatkins) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "nwatkins", "name": "nwatkins", "remove": false, "state": "absent"} 2021-11-11T20:39:40.135 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item=sidharthanup) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "sidharthanup", "name": "sidharthanup", "remove": false, "state": "absent"} 2021-11-11T20:39:40.301 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item=sidharthanup) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "sidharthanup", "name": "sidharthanup", "remove": false, "state": "absent"} 2021-11-11T20:39:41.633 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item=varsha) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "varsha", "name": "varsha", "remove": false, "state": "absent"} 2021-11-11T20:39:41.810 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item=varsha) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "varsha", "name": "varsha", "remove": false, "state": "absent"} 2021-11-11T20:39:43.215 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item=hmunjulu) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "hmunjulu", "name": "hmunjulu", "remove": false, "state": "absent"} 2021-11-11T20:39:43.288 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item=hmunjulu) => {"ansible_loop_var": "item", "changed": true, "force": false, "item": "hmunjulu", "name": "hmunjulu", "remove": false, "state": "absent"} 2021-11-11T20:39:43.414 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=nlevine) => {"ansible_loop_var": "item", "changed": false, "item": "nlevine", "name": "nlevine", "state": "absent"} 2021-11-11T20:39:43.498 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=nlevine) => {"ansible_loop_var": "item", "changed": false, "item": "nlevine", "name": "nlevine", "state": "absent"} 2021-11-11T20:39:43.614 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=tbrekke) => {"ansible_loop_var": "item", "changed": false, "item": "tbrekke", "name": "tbrekke", "state": "absent"} 2021-11-11T20:39:43.697 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=tbrekke) => {"ansible_loop_var": "item", "changed": false, "item": "tbrekke", "name": "tbrekke", "state": "absent"} 2021-11-11T20:39:43.802 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=taco) => {"ansible_loop_var": "item", "changed": false, "item": "taco", "name": "taco", "state": "absent"} 2021-11-11T20:39:43.886 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=taco) => {"ansible_loop_var": "item", "changed": false, "item": "taco", "name": "taco", "state": "absent"} 2021-11-11T20:39:44.000 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=louis) => {"ansible_loop_var": "item", "changed": false, "item": "louis", "name": "louis", "state": "absent"} 2021-11-11T20:39:44.073 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=louis) => {"ansible_loop_var": "item", "changed": false, "item": "louis", "name": "louis", "state": "absent"} 2021-11-11T20:39:44.188 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=amarango) => {"ansible_loop_var": "item", "changed": false, "item": "amarango", "name": "amarango", "state": "absent"} 2021-11-11T20:39:44.261 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=amarango) => {"ansible_loop_var": "item", "changed": false, "item": "amarango", "name": "amarango", "state": "absent"} 2021-11-11T20:39:44.385 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=oobe) => {"ansible_loop_var": "item", "changed": false, "item": "oobe", "name": "oobe", "state": "absent"} 2021-11-11T20:39:44.448 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=oobe) => {"ansible_loop_var": "item", "changed": false, "item": "oobe", "name": "oobe", "state": "absent"} 2021-11-11T20:39:44.573 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=rturk) => {"ansible_loop_var": "item", "changed": false, "item": "rturk", "name": "rturk", "state": "absent"} 2021-11-11T20:39:44.636 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=rturk) => {"ansible_loop_var": "item", "changed": false, "item": "rturk", "name": "rturk", "state": "absent"} 2021-11-11T20:39:44.761 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=fche) => {"ansible_loop_var": "item", "changed": false, "item": "fche", "name": "fche", "state": "absent"} 2021-11-11T20:39:44.835 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=fche) => {"ansible_loop_var": "item", "changed": false, "item": "fche", "name": "fche", "state": "absent"} 2021-11-11T20:39:44.949 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=jbainbri) => {"ansible_loop_var": "item", "changed": false, "item": "jbainbri", "name": "jbainbri", "state": "absent"} 2021-11-11T20:39:45.022 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=jbainbri) => {"ansible_loop_var": "item", "changed": false, "item": "jbainbri", "name": "jbainbri", "state": "absent"} 2021-11-11T20:39:45.146 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=kdhananj) => {"ansible_loop_var": "item", "changed": false, "item": "kdhananj", "name": "kdhananj", "state": "absent"} 2021-11-11T20:39:45.190 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:45 +0000 (0:00:18.803) 0:02:05.446 ***** 2021-11-11T20:39:45.218 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=kdhananj) => {"ansible_loop_var": "item", "changed": false, "item": "kdhananj", "name": "kdhananj", "state": "absent"} 2021-11-11T20:39:45.269 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:45 +0000 (0:00:00.079) 0:02:05.525 ***** 2021-11-11T20:39:45.722 INFO:teuthology.task.ansible.out: TASK [users : Write /keys-repo-sha1 sentinel file] ***************************** 2021-11-11T20:39:45.723 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "checksum": "b3b89f3dabc35f78585fcbe6afcac330df9308f6", "dest": "/keys-repo-sha1", "gid": 0, "group": "root", "md5sum": "69c3df528743245bb36ff3b8fc484a02", "mode": "0644", "owner": "root", "secontext": "system_u:object_r:etc_runtime_t:s0", "size": 40, "src": "/home/ubuntu/.ansible/tmp/ansible-tmp-1636663185.2847645-9057-121064018148643/source", "state": "file", "uid": 0} 2021-11-11T20:39:45.745 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:45 +0000 (0:00:00.475) 0:02:06.001 ***** 2021-11-11T20:39:45.792 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "checksum": "b3b89f3dabc35f78585fcbe6afcac330df9308f6", "dest": "/keys-repo-sha1", "gid": 0, "group": "root", "md5sum": "69c3df528743245bb36ff3b8fc484a02", "mode": "0644", "owner": "root", "secontext": "system_u:object_r:etc_runtime_t:s0", "size": 40, "src": "/home/ubuntu/.ansible/tmp/ansible-tmp-1636663185.35762-9075-145471925798227/source", "state": "file", "uid": 0} 2021-11-11T20:39:45.815 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:45 +0000 (0:00:00.069) 0:02:06.071 ***** 2021-11-11T20:39:45.820 INFO:teuthology.task.ansible.out: TASK [common : Log the OS name, version and release] *************************** 2021-11-11T20:39:45.822 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => { "msg": "Host smithi198.front.sepia.ceph.com is running RedHat 8.4 (Ootpa)" } 2021-11-11T20:39:45.847 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:45 +0000 (0:00:00.032) 0:02:06.103 ***** 2021-11-11T20:39:45.875 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => { "msg": "Host smithi096.front.sepia.ceph.com is running RedHat 8.4 (Ootpa)" } 2021-11-11T20:39:45.897 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:45 +0000 (0:00:00.049) 0:02:06.153 ***** 2021-11-11T20:39:45.926 INFO:teuthology.task.ansible.out: TASK [common : Including major version specific variables.] ******************** 2021-11-11T20:39:45.928 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/common/vars/redhat_8.yml) => {"ansible_facts": {"nrpe_selinux_packages": ["python3-libsemanage", "python3-policycoreutils"], "rhsm_repos": ["rhel-8-for-x86_64-baseos-rpms", "rhel-8-for-x86_64-appstream-rpms"]}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/common/vars/redhat_8.yml"], "ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/common/vars/redhat_8.yml"} 2021-11-11T20:39:45.951 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:45 +0000 (0:00:00.053) 0:02:06.207 ***** 2021-11-11T20:39:45.967 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/common/vars/redhat_8.yml) => {"ansible_facts": {"nrpe_selinux_packages": ["python3-libsemanage", "python3-policycoreutils"], "rhsm_repos": ["rhel-8-for-x86_64-baseos-rpms", "rhel-8-for-x86_64-appstream-rpms"]}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/common/vars/redhat_8.yml"], "ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/common/vars/redhat_8.yml"} 2021-11-11T20:39:45.991 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:45 +0000 (0:00:00.040) 0:02:06.247 ***** 2021-11-11T20:39:46.018 INFO:teuthology.task.ansible.out: TASK [common : Get the current timezone (RHEL/CentOS 6)] *********************** 2021-11-11T20:39:46.020 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.041 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.050) 0:02:06.297 ***** 2021-11-11T20:39:46.058 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.081 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.039) 0:02:06.337 ***** 2021-11-11T20:39:46.326 INFO:teuthology.task.ansible.out: TASK [common : Get the current timezone (RHEL/CentOS 7)] *********************** 2021-11-11T20:39:46.328 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "cmd": "timedatectl | grep -E \"Time ?zone\" | sed -e \"s/.*: \\(.*\\) (.*/\\1/\"", "delta": "0:00:00.062352", "end": "2021-11-11 20:39:46.309252", "rc": 0, "start": "2021-11-11 20:39:46.246900", "stderr": "", "stderr_lines": [], "stdout": "Etc/UTC", "stdout_lines": ["Etc/UTC"]} 2021-11-11T20:39:46.351 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.270) 0:02:06.607 ***** 2021-11-11T20:39:46.368 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "cmd": "timedatectl | grep -E \"Time ?zone\" | sed -e \"s/.*: \\(.*\\) (.*/\\1/\"", "delta": "0:00:00.054892", "end": "2021-11-11 20:39:46.343747", "rc": 0, "start": "2021-11-11 20:39:46.288855", "stderr": "", "stderr_lines": [], "stdout": "Etc/UTC", "stdout_lines": ["Etc/UTC"]} 2021-11-11T20:39:46.393 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.041) 0:02:06.648 ***** 2021-11-11T20:39:46.411 INFO:teuthology.task.ansible.out: TASK [common : Reboot RHEL7 to workaround systemd bug] ************************* 2021-11-11T20:39:46.412 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.437 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.044) 0:02:06.693 ***** 2021-11-11T20:39:46.465 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.488 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.051) 0:02:06.744 ***** 2021-11-11T20:39:46.506 INFO:teuthology.task.ansible.out: TASK [common : Wait for reboot in case of systemd workaround] ****************** 2021-11-11T20:39:46.508 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.530 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.041) 0:02:06.786 ***** 2021-11-11T20:39:46.559 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.583 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.052) 0:02:06.839 ***** 2021-11-11T20:39:46.599 INFO:teuthology.task.ansible.out: TASK [common : Set /etc/localtime (RHEL/CentOS 6)] ***************************** 2021-11-11T20:39:46.601 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.622 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.039) 0:02:06.878 ***** 2021-11-11T20:39:46.649 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.672 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.049) 0:02:06.928 ***** 2021-11-11T20:39:46.689 INFO:teuthology.task.ansible.out: TASK [common : Set the timezone (RHEL/CentOS >= 7)] **************************** 2021-11-11T20:39:46.690 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.717 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.044) 0:02:06.973 ***** 2021-11-11T20:39:46.755 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.778 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.061) 0:02:07.034 ***** 2021-11-11T20:39:46.787 INFO:teuthology.task.ansible.out: TASK [common : Install yum on Fedora 22 and later] ***************************** 2021-11-11T20:39:46.789 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.819 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.040) 0:02:07.075 ***** 2021-11-11T20:39:46.864 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:39:46.887 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:46 +0000 (0:00:00.068) 0:02:07.143 ***** 2021-11-11T20:39:46.915 INFO:teuthology.task.ansible.out: TASK [common : Set entitlements_path] ****************************************** 2021-11-11T20:39:47.048 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"entitlements_path": "/etc/ansible/secrets/entitlements.yml"}, "changed": false} Thursday 11 November 2021 20:39:46 +0000 (0:00:00.051) 0:02:07.195 ***** ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"entitlements_path": "/etc/ansible/secrets/entitlements.yml"}, "changed": false} Thursday 11 November 2021 20:39:46 +0000 (0:00:00.050) 0:02:07.245 ***** TASK [common : Include Red Hat encrypted variables.] *************************** ok: [smithi198.front.sepia.ceph.com] => (item=None) => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false} ok: [smithi198.front.sepia.ceph.com] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false} 2021-11-11T20:39:47.051 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:47 +0000 (0:00:00.061) 0:02:07.307 ***** 2021-11-11T20:39:47.078 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=None) => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false} 2021-11-11T20:39:47.080 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false} 2021-11-11T20:39:47.101 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:47 +0000 (0:00:00.049) 0:02:07.357 ***** 2021-11-11T20:39:47.128 INFO:teuthology.task.ansible.out: TASK [common : Set have_entitlements] ****************************************** 2021-11-11T20:39:47.130 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"have_entitlements": true}, "changed": false} 2021-11-11T20:39:47.152 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:47 +0000 (0:00:00.051) 0:02:07.408 ***** 2021-11-11T20:39:47.180 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"have_entitlements": true}, "changed": false} 2021-11-11T20:39:47.202 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:47 +0000 (0:00:00.049) 0:02:07.458 ***** 2021-11-11T20:39:48.442 INFO:teuthology.task.ansible.out: TASK [common : Install CA Cert from Satellite Server] ************************** 2021-11-11T20:39:48.445 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": ["Installed /root/.ansible/tmp/ansible-moduletmp-1636663188.2836497-lurh8y0_/katello-ca-consumer-latest.noarch2jhokkfg.rpm"]} ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": ["Installed /root/.ansible/tmp/ansible-moduletmp-1636663188.2860956-_w_exs7x/katello-ca-consumer-latest.noarchymgck05f.rpm"]} 2021-11-11T20:39:48.464 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:48 +0000 (0:00:01.262) 0:02:08.720 ***** 2021-11-11T20:39:48.484 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:48 +0000 (0:00:00.019) 0:02:08.740 ***** 2021-11-11T20:39:49.030 INFO:teuthology.task.ansible.out: TASK [common : Set the releasever] ********************************************* 2021-11-11T20:39:49.031 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "532cb5b70a3f71ce180c165205647e8909d4e8a0", "dest": "/etc/yum/vars/releasever", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/yum/vars/releasever", "secontext": "system_u:object_r:etc_t:s0", "size": 3, "state": "file", "uid": 0} 2021-11-11T20:39:49.052 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:49 +0000 (0:00:00.567) 0:02:09.308 ***** 2021-11-11T20:39:49.059 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "532cb5b70a3f71ce180c165205647e8909d4e8a0", "dest": "/etc/yum/vars/releasever", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/yum/vars/releasever", "secontext": "system_u:object_r:etc_t:s0", "size": 3, "state": "file", "uid": 0} 2021-11-11T20:39:49.083 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:49 +0000 (0:00:00.031) 0:02:09.339 ***** 2021-11-11T20:39:49.710 INFO:teuthology.task.ansible.out: TASK [common : Determine if node is registered with subscription-manager.] ***** 2021-11-11T20:39:49.712 INFO:teuthology.task.ansible.out:fatal: [smithi198.front.sepia.ceph.com]: FAILED! => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false} ...ignoring fatal: [smithi096.front.sepia.ceph.com]: FAILED! => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false} ...ignoring 2021-11-11T20:39:49.732 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:49 +0000 (0:00:00.648) 0:02:09.988 ***** 2021-11-11T20:39:49.751 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:49 +0000 (0:00:00.018) 0:02:10.007 ***** 2021-11-11T20:39:49.810 INFO:teuthology.task.ansible.out: TASK [common : Set rhsm_registered if we're already registered] **************** 2021-11-11T20:39:49.811 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"rhsm_registered": false}, "changed": false} 2021-11-11T20:39:49.832 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:49 +0000 (0:00:00.081) 0:02:10.088 ***** 2021-11-11T20:39:49.839 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"rhsm_registered": false}, "changed": false} 2021-11-11T20:39:49.863 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:39:49 +0000 (0:00:00.030) 0:02:10.119 ***** 2021-11-11T20:40:08.827 INFO:teuthology.task.ansible.out: TASK [common : Register with subscription-manager.] **************************** 2021-11-11T20:40:08.829 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"attempts": 1, "censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": true} 2021-11-11T20:40:08.850 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:08 +0000 (0:00:18.987) 0:02:29.106 ***** 2021-11-11T20:40:08.919 INFO:teuthology.task.ansible.out: TASK [common : Set rhsm_registered if we just registered] ********************** 2021-11-11T20:40:08.921 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"rhsm_registered": true}, "changed": false} 2021-11-11T20:40:08.942 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:08 +0000 (0:00:00.091) 0:02:29.198 ***** 2021-11-11T20:40:09.603 INFO:teuthology.task.ansible.out: TASK [common : List CDN releases available to system] ************************** 2021-11-11T20:40:09.605 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "cmd": "subscription-manager release --list | grep -E '[0-9]'", "delta": "0:00:00.430311", "end": "2021-11-11 20:40:09.572443", "failed_when_result": false, "rc": 0, "start": "2021-11-11 20:40:09.142132", "stderr": "", "stderr_lines": [], "stdout": "7.8\n7.9\n7Server\n8\n8.2\n8.3\n8.4", "stdout_lines": ["7.8", "7.9", "7Server", "8", "8.2", "8.3", "8.4"]} 2021-11-11T20:40:09.624 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:09 +0000 (0:00:00.682) 0:02:29.880 ***** 2021-11-11T20:40:09.725 INFO:teuthology.task.ansible.out: TASK [common : Unregister Beta/Alpha/RC system with subscription-manager] ****** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:09.746 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:09 +0000 (0:00:00.122) 0:02:30.002 ***** 2021-11-11T20:40:09.826 INFO:teuthology.task.ansible.out: TASK [common : Set rhsm_registered to false if Beta/Alpha/RC release] ********** 2021-11-11T20:40:09.828 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:09.858 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:09 +0000 (0:00:00.112) 0:02:30.114 ***** 2021-11-11T20:40:09.880 INFO:teuthology.task.ansible.out: TASK [common : Register with subscription-manager.] **************************** 2021-11-11T20:40:09.882 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"attempts": 1, "censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": true} 2021-11-11T20:40:09.908 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:09 +0000 (0:00:00.049) 0:02:30.164 ***** 2021-11-11T20:40:09.925 INFO:teuthology.task.ansible.out: TASK [common : Configure internal beta repos in /etc/yum.repos.d/] ************* 2021-11-11T20:40:09.927 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false} 2021-11-11T20:40:09.948 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:09 +0000 (0:00:00.040) 0:02:30.204 ***** 2021-11-11T20:40:09.986 INFO:teuthology.task.ansible.out: TASK [common : Set rhsm_registered if we just registered] ********************** 2021-11-11T20:40:09.988 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"rhsm_registered": true}, "changed": false} 2021-11-11T20:40:10.009 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:10 +0000 (0:00:00.061) 0:02:30.265 ***** 2021-11-11T20:40:10.027 INFO:teuthology.task.ansible.out: TASK [common : Clean yum cache] ************************************************ 2021-11-11T20:40:10.028 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:10.049 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:10 +0000 (0:00:00.039) 0:02:30.305 ***** 2021-11-11T20:40:10.693 INFO:teuthology.task.ansible.out: TASK [common : List CDN releases available to system] ************************** 2021-11-11T20:40:10.695 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "cmd": "subscription-manager release --list | grep -E '[0-9]'", "delta": "0:00:00.446074", "end": "2021-11-11 20:40:10.666413", "failed_when_result": false, "rc": 0, "start": "2021-11-11 20:40:10.220339", "stderr": "", "stderr_lines": [], "stdout": "7.8\n7.9\n7Server\n8\n8.2\n8.3\n8.4", "stdout_lines": ["7.8", "7.9", "7Server", "8", "8.2", "8.3", "8.4"]} 2021-11-11T20:40:10.716 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:10 +0000 (0:00:00.666) 0:02:30.972 ***** 2021-11-11T20:40:10.785 INFO:teuthology.task.ansible.out: TASK [common : Unregister Beta/Alpha/RC system with subscription-manager] ****** 2021-11-11T20:40:10.787 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:10.806 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:10 +0000 (0:00:00.090) 0:02:31.062 ***** 2021-11-11T20:40:10.875 INFO:teuthology.task.ansible.out: TASK [common : Set rhsm_registered to false if Beta/Alpha/RC release] ********** 2021-11-11T20:40:10.876 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:10.895 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:10 +0000 (0:00:00.089) 0:02:31.152 ***** 2021-11-11T20:40:10.953 INFO:teuthology.task.ansible.out: TASK [common : Configure internal beta repos in /etc/yum.repos.d/] ************* 2021-11-11T20:40:10.955 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false} 2021-11-11T20:40:10.975 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:10 +0000 (0:00:00.079) 0:02:31.231 ***** 2021-11-11T20:40:11.043 INFO:teuthology.task.ansible.out: TASK [common : Clean yum cache] ************************************************ 2021-11-11T20:40:11.045 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:11.070 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:11 +0000 (0:00:00.095) 0:02:31.326 ***** 2021-11-11T20:40:13.094 INFO:teuthology.task.ansible.out: TASK [common : Get list of enabled RHSM repos] ********************************* 2021-11-11T20:40:13.096 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "cmd": "subscription-manager repos --list | grep -B4 'Enabled:.*1' | grep 'Repo ID:' | sed -e 's/Repo ID:\\s*\\(.*\\)/\\1/' | sort", "delta": "0:00:02.803802", "end": "2021-11-11 20:40:13.071149", "rc": 0, "start": "2021-11-11 20:40:10.267347", "stderr": "", "stderr_lines": [], "stdout": "rhel-8-for-x86_64-appstream-rpms\nrhel-8-for-x86_64-baseos-rpms", "stdout_lines": ["rhel-8-for-x86_64-appstream-rpms", "rhel-8-for-x86_64-baseos-rpms"]} 2021-11-11T20:40:13.118 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:13 +0000 (0:00:02.047) 0:02:33.374 ***** 2021-11-11T20:40:13.196 INFO:teuthology.task.ansible.out: TASK [common : Store list of enabled repos] ************************************ 2021-11-11T20:40:13.198 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"repo_list": ["rhel-8-for-x86_64-appstream-rpms", "rhel-8-for-x86_64-baseos-rpms"]}, "changed": false} 2021-11-11T20:40:13.218 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:13 +0000 (0:00:00.100) 0:02:33.474 ***** 2021-11-11T20:40:13.286 INFO:teuthology.task.ansible.out: TASK [common : Set replace_repos false if entitlements are missing or if we unregistered] *** 2021-11-11T20:40:13.288 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:13.308 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:13 +0000 (0:00:00.090) 0:02:33.564 ***** 2021-11-11T20:40:13.387 INFO:teuthology.task.ansible.out: TASK [common : Set replace_repos true if rhsm_repos differs from repo_list] **** 2021-11-11T20:40:13.389 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"replace_repos": false}, "changed": false} 2021-11-11T20:40:13.409 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:13 +0000 (0:00:00.100) 0:02:33.665 ***** 2021-11-11T20:40:13.488 INFO:teuthology.task.ansible.out: TASK [common : Set replace_repos true if newly-subscribed] ********************* 2021-11-11T20:40:13.490 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"replace_repos": true}, "changed": false} 2021-11-11T20:40:13.511 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:13 +0000 (0:00:00.101) 0:02:33.767 ***** 2021-11-11T20:40:14.211 INFO:teuthology.task.ansible.out: TASK [common : Get list of enabled RHSM repos] ********************************* 2021-11-11T20:40:14.213 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "cmd": "subscription-manager repos --list | grep -B4 'Enabled:.*1' | grep 'Repo ID:' | sed -e 's/Repo ID:\\s*\\(.*\\)/\\1/' | sort", "delta": "0:00:02.907236", "end": "2021-11-11 20:40:14.189580", "rc": 0, "start": "2021-11-11 20:40:11.282344", "stderr": "", "stderr_lines": [], "stdout": "rhel-8-for-x86_64-appstream-rpms\nrhel-8-for-x86_64-baseos-rpms", "stdout_lines": ["rhel-8-for-x86_64-appstream-rpms", "rhel-8-for-x86_64-baseos-rpms"]} 2021-11-11T20:40:14.234 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:14 +0000 (0:00:00.723) 0:02:34.490 ***** 2021-11-11T20:40:14.313 INFO:teuthology.task.ansible.out: TASK [common : Store list of enabled repos] ************************************ 2021-11-11T20:40:14.315 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"repo_list": ["rhel-8-for-x86_64-appstream-rpms", "rhel-8-for-x86_64-baseos-rpms"]}, "changed": false} 2021-11-11T20:40:14.335 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:14 +0000 (0:00:00.100) 0:02:34.591 ***** 2021-11-11T20:40:14.413 INFO:teuthology.task.ansible.out: TASK [common : Set replace_repos false if entitlements are missing or if we unregistered] *** 2021-11-11T20:40:14.415 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:14.434 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:14 +0000 (0:00:00.099) 0:02:34.690 ***** 2021-11-11T20:40:14.512 INFO:teuthology.task.ansible.out: TASK [common : Set replace_repos true if rhsm_repos differs from repo_list] **** ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"replace_repos": false}, "changed": false} 2021-11-11T20:40:14.534 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:14 +0000 (0:00:00.099) 0:02:34.790 ***** 2021-11-11T20:40:14.612 INFO:teuthology.task.ansible.out: TASK [common : Set replace_repos true if newly-subscribed] ********************* 2021-11-11T20:40:14.614 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"replace_repos": true}, "changed": false} 2021-11-11T20:40:14.634 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:14 +0000 (0:00:00.100) 0:02:34.890 ***** 2021-11-11T20:40:15.723 INFO:teuthology.task.ansible.out: TASK [common : Disable all rhsm repos] ***************************************** 2021-11-11T20:40:15.725 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": true} 2021-11-11T20:40:15.745 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:15 +0000 (0:00:01.110) 0:02:36.001 ***** 2021-11-11T20:40:17.303 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": true} 2021-11-11T20:40:17.326 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:17 +0000 (0:00:01.581) 0:02:37.582 ***** 2021-11-11T20:40:19.508 INFO:teuthology.task.ansible.out: TASK [common : Enable necessary rhsm repos] ************************************ 2021-11-11T20:40:19.510 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": ["subscription-manager", "repos", "--enable=rhel-8-for-x86_64-baseos-rpms", "--enable=rhel-8-for-x86_64-appstream-rpms"], "delta": "0:00:03.530670", "end": "2021-11-11 20:40:19.487433", "rc": 0, "start": "2021-11-11 20:40:15.956763", "stderr": "", "stderr_lines": [], "stdout": "Repository 'rhel-8-for-x86_64-baseos-rpms' is enabled for this system.\nRepository 'rhel-8-for-x86_64-appstream-rpms' is enabled for this system.", "stdout_lines": ["Repository 'rhel-8-for-x86_64-baseos-rpms' is enabled for this system.", "Repository 'rhel-8-for-x86_64-appstream-rpms' is enabled for this system."]} 2021-11-11T20:40:19.529 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:19 +0000 (0:00:02.203) 0:02:39.785 ***** 2021-11-11T20:40:19.754 INFO:teuthology.task.ansible.out: TASK [common : Remove old apt-mirror repository definition.] ******************* 2021-11-11T20:40:19.756 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "path": "/etc/yum.repos.d/cd.repo", "state": "absent"} 2021-11-11T20:40:19.776 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:19 +0000 (0:00:00.246) 0:02:40.032 ***** 2021-11-11T20:40:20.001 INFO:teuthology.task.ansible.out: TASK [common : Increase the yum timeout.] ************************************** 2021-11-11T20:40:20.002 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"backup": "", "changed": false, "msg": ""} 2021-11-11T20:40:20.023 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:20 +0000 (0:00:00.246) 0:02:40.279 ***** 2021-11-11T20:40:20.526 INFO:teuthology.task.ansible.out: TASK [common : Configure epel repos in /etc/yum.repos.d/] ********************** 2021-11-11T20:40:20.528 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'key': 'epel', 'value': {'name': 'Extra Packages for Enterprise Linux', 'mirrorlist': 'file:///etc/yum.repos.d/epel-mirrorlist', 'failovermethod': 'priority', 'enabled': '1', 'gpgcheck': 0}}) => {"ansible_loop_var": "item", "changed": false, "checksum": "86c0d58bde7ad4cfaafe592d2fefd92c920c42ca", "dest": "/etc/yum.repos.d/epel.repo", "gid": 0, "group": "root", "item": {"key": "epel", "value": {"enabled": "1", "failovermethod": "priority", "gpgcheck": 0, "mirrorlist": "file:///etc/yum.repos.d/epel-mirrorlist", "name": "Extra Packages for Enterprise Linux"}}, "mode": "0644", "owner": "root", "path": "/etc/yum.repos.d/epel.repo", "secontext": "system_u:object_r:system_conf_t:s0", "size": 236, "state": "file", "uid": 0} 2021-11-11T20:40:20.952 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'key': 'epel-testing', 'value': {'name': 'Extra Packages for Enterprise Linux - Testing', 'mirrorlist': 'file:///etc/yum.repos.d/epel-testing-mirrorlist', 'failovermethod': 'priority', 'enabled': 0, 'gpgcheck': 0}}) => {"ansible_loop_var": "item", "changed": false, "checksum": "a1bad5509ed35b895c2061841236b1cfe10619ae", "dest": "/etc/yum.repos.d/epel-testing.repo", "gid": 0, "group": "root", "item": {"key": "epel-testing", "value": {"enabled": 0, "failovermethod": "priority", "gpgcheck": 0, "mirrorlist": "file:///etc/yum.repos.d/epel-testing-mirrorlist", "name": "Extra Packages for Enterprise Linux - Testing"}}, "mode": "0644", "owner": "root", "path": "/etc/yum.repos.d/epel-testing.repo", "secontext": "system_u:object_r:system_conf_t:s0", "size": 262, "state": "file", "uid": 0} 2021-11-11T20:40:20.976 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:20 +0000 (0:00:00.953) 0:02:41.232 ***** 2021-11-11T20:40:20.982 INFO:teuthology.task.ansible.out: TASK [common : Enable necessary rhsm repos] ************************************ 2021-11-11T20:40:20.984 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": ["subscription-manager", "repos", "--enable=rhel-8-for-x86_64-baseos-rpms", "--enable=rhel-8-for-x86_64-appstream-rpms"], "delta": "0:00:03.417339", "end": "2021-11-11 20:40:20.955804", "rc": 0, "start": "2021-11-11 20:40:17.538465", "stderr": "", "stderr_lines": [], "stdout": "Repository 'rhel-8-for-x86_64-baseos-rpms' is enabled for this system.\nRepository 'rhel-8-for-x86_64-appstream-rpms' is enabled for this system.", "stdout_lines": ["Repository 'rhel-8-for-x86_64-baseos-rpms' is enabled for this system.", "Repository 'rhel-8-for-x86_64-appstream-rpms' is enabled for this system."]} 2021-11-11T20:40:21.006 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:20 +0000 (0:00:00.029) 0:02:41.262 ***** 2021-11-11T20:40:21.235 INFO:teuthology.task.ansible.out: TASK [common : Remove old apt-mirror repository definition.] ******************* 2021-11-11T20:40:21.236 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "path": "/etc/yum.repos.d/cd.repo", "state": "absent"} 2021-11-11T20:40:21.256 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:21 +0000 (0:00:00.250) 0:02:41.513 ***** 2021-11-11T20:40:21.480 INFO:teuthology.task.ansible.out: TASK [common : Increase the yum timeout.] ************************************** 2021-11-11T20:40:21.482 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"backup": "", "changed": false, "msg": ""} 2021-11-11T20:40:21.503 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:21 +0000 (0:00:00.246) 0:02:41.759 ***** 2021-11-11T20:40:21.510 INFO:teuthology.task.ansible.out: TASK [common : Configure local epel mirrorlists] ******************************* 2021-11-11T20:40:21.511 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=epel-mirrorlist) => {"ansible_loop_var": "item", "changed": false, "checksum": "3c9ba7bacd1e30bed014cefd1fb4eab0f0f8fa10", "dest": "/etc/yum.repos.d/epel-mirrorlist", "gid": 0, "group": "root", "item": "epel-mirrorlist", "mode": "0644", "owner": "root", "path": "/etc/yum.repos.d/epel-mirrorlist", "secontext": "system_u:object_r:system_conf_t:s0", "size": 484, "state": "file", "uid": 0} 2021-11-11T20:40:21.918 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=epel-testing-mirrorlist) => {"ansible_loop_var": "item", "changed": false, "checksum": "d542c3e2c30a0d3eb7d454775342c302c0ded15b", "dest": "/etc/yum.repos.d/epel-testing-mirrorlist", "gid": 0, "group": "root", "item": "epel-testing-mirrorlist", "mode": "0644", "owner": "root", "path": "/etc/yum.repos.d/epel-testing-mirrorlist", "secontext": "system_u:object_r:system_conf_t:s0", "size": 540, "state": "file", "uid": 0} 2021-11-11T20:40:21.940 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:21 +0000 (0:00:00.437) 0:02:42.196 ***** 2021-11-11T20:40:22.000 INFO:teuthology.task.ansible.out: TASK [common : Configure epel repos in /etc/yum.repos.d/] ********************** 2021-11-11T20:40:22.001 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'key': 'epel', 'value': {'name': 'Extra Packages for Enterprise Linux', 'mirrorlist': 'file:///etc/yum.repos.d/epel-mirrorlist', 'failovermethod': 'priority', 'enabled': '1', 'gpgcheck': 0}}) => {"ansible_loop_var": "item", "changed": false, "checksum": "86c0d58bde7ad4cfaafe592d2fefd92c920c42ca", "dest": "/etc/yum.repos.d/epel.repo", "gid": 0, "group": "root", "item": {"key": "epel", "value": {"enabled": "1", "failovermethod": "priority", "gpgcheck": 0, "mirrorlist": "file:///etc/yum.repos.d/epel-mirrorlist", "name": "Extra Packages for Enterprise Linux"}}, "mode": "0644", "owner": "root", "path": "/etc/yum.repos.d/epel.repo", "secontext": "system_u:object_r:system_conf_t:s0", "size": 236, "state": "file", "uid": 0} 2021-11-11T20:40:22.011 INFO:teuthology.task.ansible.out: TASK [common : Clean yum cache] ************************************************ 2021-11-11T20:40:22.013 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.032 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:22 +0000 (0:00:00.091) 0:02:42.288 ***** 2021-11-11T20:40:22.100 INFO:teuthology.task.ansible.out: TASK [common : Get the current timezone] *************************************** 2021-11-11T20:40:22.102 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.122 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:22 +0000 (0:00:00.089) 0:02:42.378 ***** 2021-11-11T20:40:22.210 INFO:teuthology.task.ansible.out: TASK [common : Set the timezone in /etc/timezone] ****************************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.232 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:22 +0000 (0:00:00.109) 0:02:42.488 ***** 2021-11-11T20:40:22.290 INFO:teuthology.task.ansible.out: TASK [common : Inform the OS of the new timezone] ****************************** 2021-11-11T20:40:22.291 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.311 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:22 +0000 (0:00:00.078) 0:02:42.567 ***** 2021-11-11T20:40:22.379 INFO:teuthology.task.ansible.out: TASK [common : Mask sleep units] *********************************************** 2021-11-11T20:40:22.381 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=sleep.target) => {"ansible_loop_var": "item", "changed": false, "item": "sleep.target", "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item=suspend.target) => {"ansible_loop_var": "item", "changed": false, "item": "suspend.target", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.390 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=hibernate.target) => {"ansible_loop_var": "item", "changed": false, "item": "hibernate.target", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.392 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=hybrid-sleep.target) => {"ansible_loop_var": "item", "changed": false, "item": "hybrid-sleep.target", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.414 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:22 +0000 (0:00:00.103) 0:02:42.670 ***** 2021-11-11T20:40:22.431 INFO:teuthology.task.ansible.out: TASK [common : Configure epel repos in /etc/yum.repos.d/] ********************** 2021-11-11T20:40:22.433 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'key': 'epel-testing', 'value': {'name': 'Extra Packages for Enterprise Linux - Testing', 'mirrorlist': 'file:///etc/yum.repos.d/epel-testing-mirrorlist', 'failovermethod': 'priority', 'enabled': 0, 'gpgcheck': 0}}) => {"ansible_loop_var": "item", "changed": false, "checksum": "a1bad5509ed35b895c2061841236b1cfe10619ae", "dest": "/etc/yum.repos.d/epel-testing.repo", "gid": 0, "group": "root", "item": {"key": "epel-testing", "value": {"enabled": 0, "failovermethod": "priority", "gpgcheck": 0, "mirrorlist": "file:///etc/yum.repos.d/epel-testing-mirrorlist", "name": "Extra Packages for Enterprise Linux - Testing"}}, "mode": "0644", "owner": "root", "path": "/etc/yum.repos.d/epel-testing.repo", "secontext": "system_u:object_r:system_conf_t:s0", "size": 262, "state": "file", "uid": 0} 2021-11-11T20:40:22.457 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:22 +0000 (0:00:00.042) 0:02:42.713 ***** 2021-11-11T20:40:22.474 INFO:teuthology.task.ansible.out: TASK [common : Get the current timezone] *************************************** 2021-11-11T20:40:22.476 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.496 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:22 +0000 (0:00:00.039) 0:02:42.752 ***** 2021-11-11T20:40:22.565 INFO:teuthology.task.ansible.out: TASK [common : Set the timezone] *********************************************** 2021-11-11T20:40:22.566 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.586 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:22 +0000 (0:00:00.089) 0:02:42.842 ***** 2021-11-11T20:40:22.654 INFO:teuthology.task.ansible.out: TASK [common : Add base OpenSUSE Leap repo] ************************************ skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.675 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:22 +0000 (0:00:00.088) 0:02:42.931 ***** 2021-11-11T20:40:22.742 INFO:teuthology.task.ansible.out: TASK [common : Add updates OpenSUSE Leap repo] ********************************* 2021-11-11T20:40:22.744 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.764 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:22 +0000 (0:00:00.089) 0:02:43.020 ***** 2021-11-11T20:40:22.832 INFO:teuthology.task.ansible.out: TASK [common : Refresh repos] ************************************************** 2021-11-11T20:40:22.834 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:22.853 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:22 +0000 (0:00:00.088) 0:02:43.109 ***** 2021-11-11T20:40:22.962 INFO:teuthology.task.ansible.out: TASK [common : Configure local epel mirrorlists] ******************************* 2021-11-11T20:40:22.964 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=epel-mirrorlist) => {"ansible_loop_var": "item", "changed": false, "checksum": "3c9ba7bacd1e30bed014cefd1fb4eab0f0f8fa10", "dest": "/etc/yum.repos.d/epel-mirrorlist", "gid": 0, "group": "root", "item": "epel-mirrorlist", "mode": "0644", "owner": "root", "path": "/etc/yum.repos.d/epel-mirrorlist", "secontext": "system_u:object_r:system_conf_t:s0", "size": 484, "state": "file", "uid": 0} 2021-11-11T20:40:23.405 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=epel-testing-mirrorlist) => {"ansible_loop_var": "item", "changed": false, "checksum": "d542c3e2c30a0d3eb7d454775342c302c0ded15b", "dest": "/etc/yum.repos.d/epel-testing-mirrorlist", "gid": 0, "group": "root", "item": "epel-testing-mirrorlist", "mode": "0644", "owner": "root", "path": "/etc/yum.repos.d/epel-testing-mirrorlist", "secontext": "system_u:object_r:system_conf_t:s0", "size": 540, "state": "file", "uid": 0} 2021-11-11T20:40:23.427 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:23 +0000 (0:00:00.574) 0:02:43.683 ***** 2021-11-11T20:40:23.496 INFO:teuthology.task.ansible.out: TASK [common : Clean yum cache] ************************************************ 2021-11-11T20:40:23.497 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:23.516 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:23 +0000 (0:00:00.088) 0:02:43.772 ***** 2021-11-11T20:40:23.522 INFO:teuthology.task.ansible.out: TASK [common : Set the hardware clock] ***************************************** 2021-11-11T20:40:23.523 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": ["hwclock", "--systohc"], "delta": "0:00:00.481386", "end": "2021-11-11 20:40:23.500315", "rc": 0, "start": "2021-11-11 20:40:23.018929", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:40:23.547 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:23 +0000 (0:00:00.031) 0:02:43.803 ***** 2021-11-11T20:40:23.574 INFO:teuthology.task.ansible.out: TASK [common : Get the current timezone] *************************************** 2021-11-11T20:40:23.576 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:23.597 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:23 +0000 (0:00:00.049) 0:02:43.853 ***** 2021-11-11T20:40:23.653 INFO:teuthology.task.ansible.out: TASK [common : Set the timezone in /etc/timezone] ****************************** 2021-11-11T20:40:23.655 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:23.674 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:23 +0000 (0:00:00.076) 0:02:43.930 ***** 2021-11-11T20:40:23.730 INFO:teuthology.task.ansible.out: TASK [common : Inform the OS of the new timezone] ****************************** 2021-11-11T20:40:23.732 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:23.751 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:23 +0000 (0:00:00.077) 0:02:44.007 ***** 2021-11-11T20:40:23.818 INFO:teuthology.task.ansible.out: TASK [common : Mask sleep units] *********************************************** 2021-11-11T20:40:23.820 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=sleep.target) => {"ansible_loop_var": "item", "changed": false, "item": "sleep.target", "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item=suspend.target) => {"ansible_loop_var": "item", "changed": false, "item": "suspend.target", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:23.830 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=hibernate.target) => {"ansible_loop_var": "item", "changed": false, "item": "hibernate.target", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:23.831 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=hybrid-sleep.target) => {"ansible_loop_var": "item", "changed": false, "item": "hybrid-sleep.target", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:23.851 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:23 +0000 (0:00:00.099) 0:02:44.107 ***** 2021-11-11T20:40:23.908 INFO:teuthology.task.ansible.out: TASK [common : Get the current timezone] *************************************** 2021-11-11T20:40:23.910 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:23.929 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:23 +0000 (0:00:00.077) 0:02:44.185 ***** 2021-11-11T20:40:23.995 INFO:teuthology.task.ansible.out: TASK [common : Set the timezone] *********************************************** 2021-11-11T20:40:23.997 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:24.016 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:24 +0000 (0:00:00.087) 0:02:44.272 ***** 2021-11-11T20:40:24.083 INFO:teuthology.task.ansible.out: TASK [common : Add base OpenSUSE Leap repo] ************************************ 2021-11-11T20:40:24.085 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:24.105 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:24 +0000 (0:00:00.089) 0:02:44.361 ***** 2021-11-11T20:40:24.173 INFO:teuthology.task.ansible.out: TASK [common : Add updates OpenSUSE Leap repo] ********************************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:24.195 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:24 +0000 (0:00:00.089) 0:02:44.451 ***** 2021-11-11T20:40:24.262 INFO:teuthology.task.ansible.out: TASK [common : Refresh repos] ************************************************** 2021-11-11T20:40:24.264 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:24.284 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:24 +0000 (0:00:00.089) 0:02:44.540 ***** 2021-11-11T20:40:24.518 INFO:teuthology.task.ansible.out: TASK [common : Set the hardware clock] ***************************************** 2021-11-11T20:40:24.520 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": ["hwclock", "--systohc"], "delta": "0:00:00.047920", "end": "2021-11-11 20:40:24.500332", "rc": 0, "start": "2021-11-11 20:40:24.452412", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:40:24.540 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:24 +0000 (0:00:00.255) 0:02:44.796 ***** 2021-11-11T20:40:37.989 INFO:teuthology.task.ansible.out: TASK [common : Install Kerberos Packages (RedHat)] ***************************** 2021-11-11T20:40:37.992 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:40:38.013 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:38 +0000 (0:00:13.473) 0:02:58.269 ***** 2021-11-11T20:40:38.096 INFO:teuthology.task.ansible.out: TASK [common : Clean apt cache] ************************************************ 2021-11-11T20:40:38.098 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:38.127 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:38 +0000 (0:00:00.113) 0:02:58.383 ***** 2021-11-11T20:40:38.187 INFO:teuthology.task.ansible.out: TASK [common : Update apt cache] *********************************************** 2021-11-11T20:40:38.189 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:38.210 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:38 +0000 (0:00:00.082) 0:02:58.466 ***** 2021-11-11T20:40:38.278 INFO:teuthology.task.ansible.out: TASK [common : Install Kerberos Packages (Debian)] ***************************** 2021-11-11T20:40:38.279 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:38.299 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:38 +0000 (0:00:00.088) 0:02:58.555 ***** 2021-11-11T20:40:38.366 INFO:teuthology.task.ansible.out: TASK [common : Install Kerberos Packages (OpenSUSE Leap)] ********************** 2021-11-11T20:40:38.368 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:38.399 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:38 +0000 (0:00:00.099) 0:02:58.655 ***** 2021-11-11T20:40:38.887 INFO:teuthology.task.ansible.out: TASK [common : Add krb5 config file] ******************************************* 2021-11-11T20:40:38.888 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "b3c1a33d19a4e448c29374429629dff57657b576", "dest": "/etc/krb5.conf", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/krb5.conf", "secontext": "system_u:object_r:krb5_conf_t:s0", "size": 131, "state": "file", "uid": 0} 2021-11-11T20:40:38.909 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:38 +0000 (0:00:00.510) 0:02:59.165 ***** 2021-11-11T20:40:39.311 INFO:teuthology.task.ansible.out: TASK [common : Install Kerberos Packages (RedHat)] ***************************** 2021-11-11T20:40:39.313 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:40:39.334 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:39 +0000 (0:00:00.424) 0:02:59.590 ***** 2021-11-11T20:40:39.402 INFO:teuthology.task.ansible.out: TASK [common : Clean apt cache] ************************************************ 2021-11-11T20:40:39.404 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:39.423 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:39 +0000 (0:00:00.089) 0:02:59.679 ***** 2021-11-11T20:40:39.491 INFO:teuthology.task.ansible.out: TASK [common : Update apt cache] *********************************************** 2021-11-11T20:40:39.493 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:39.515 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:39 +0000 (0:00:00.091) 0:02:59.771 ***** 2021-11-11T20:40:39.573 INFO:teuthology.task.ansible.out: TASK [common : Install Kerberos Packages (Debian)] ***************************** 2021-11-11T20:40:39.574 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:39.594 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:39 +0000 (0:00:00.079) 0:02:59.850 ***** 2021-11-11T20:40:39.651 INFO:teuthology.task.ansible.out: TASK [common : Install Kerberos Packages (OpenSUSE Leap)] ********************** 2021-11-11T20:40:39.653 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:39.673 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:39 +0000 (0:00:00.078) 0:02:59.929 ***** 2021-11-11T20:40:40.154 INFO:teuthology.task.ansible.out: TASK [common : Add krb5 config file] ******************************************* 2021-11-11T20:40:40.156 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "b3c1a33d19a4e448c29374429629dff57657b576", "dest": "/etc/krb5.conf", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/krb5.conf", "secontext": "system_u:object_r:krb5_conf_t:s0", "size": 131, "state": "file", "uid": 0} 2021-11-11T20:40:40.175 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:40 +0000 (0:00:00.502) 0:03:00.431 ***** 2021-11-11T20:40:41.926 INFO:teuthology.task.ansible.out: TASK [common : Make sure smartmontools is installed] *************************** 2021-11-11T20:40:41.927 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:40:41.948 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:41 +0000 (0:00:01.772) 0:03:02.204 ***** 2021-11-11T20:40:42.451 INFO:teuthology.task.ansible.out: TASK [common : Upload megacli and cli64 for raid monitoring and smart.pl to /usr/sbin/.] *** 2021-11-11T20:40:42.453 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=megacli) => {"ansible_loop_var": "item", "changed": false, "checksum": "cb3ef2aa6a3e3002b234f166e739a39b1f3a1bde", "dest": "/usr/sbin/megacli", "gid": 0, "group": "root", "item": "megacli", "mode": "0755", "owner": "root", "path": "/usr/sbin/megacli", "secontext": "system_u:object_r:bin_t:s0", "size": 2716736, "state": "file", "uid": 0} 2021-11-11T20:40:42.886 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=cli64) => {"ansible_loop_var": "item", "changed": false, "checksum": "a3b8773b14e8594ffbd9fc156c09436eb0184bc7", "dest": "/usr/sbin/cli64", "gid": 0, "group": "root", "item": "cli64", "mode": "0755", "owner": "root", "path": "/usr/sbin/cli64", "secontext": "system_u:object_r:bin_t:s0", "size": 1952038, "state": "file", "uid": 0} 2021-11-11T20:40:43.129 INFO:teuthology.task.ansible.out: TASK [common : Make sure smartmontools is installed] *************************** 2021-11-11T20:40:43.131 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:40:43.151 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:43 +0000 (0:00:01.203) 0:03:03.407 ***** 2021-11-11T20:40:43.303 INFO:teuthology.task.ansible.out: TASK [common : Upload megacli and cli64 for raid monitoring and smart.pl to /usr/sbin/.] *** 2021-11-11T20:40:43.305 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=nvme) => {"ansible_loop_var": "item", "changed": false, "checksum": "98735dcacd6fec55dd92b67281b478603b7955b1", "dest": "/usr/sbin/nvme", "gid": 0, "group": "root", "item": "nvme", "mode": "0755", "owner": "root", "path": "/usr/sbin/nvme", "secontext": "system_u:object_r:bin_t:s0", "size": 580432, "state": "file", "uid": 0} 2021-11-11T20:40:43.330 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:43 +0000 (0:00:00.178) 0:03:03.586 ***** 2021-11-11T20:40:43.513 INFO:teuthology.task.ansible.out: TASK [common : Create /usr/libexec.] ******************************************* 2021-11-11T20:40:43.515 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/usr/libexec", "secontext": "system_u:object_r:bin_t:s0", "size": 4096, "state": "directory", "uid": 0} 2021-11-11T20:40:43.534 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:43 +0000 (0:00:00.204) 0:03:03.790 ***** 2021-11-11T20:40:43.654 INFO:teuthology.task.ansible.out: TASK [common : Upload megacli and cli64 for raid monitoring and smart.pl to /usr/sbin/.] *** 2021-11-11T20:40:43.655 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=megacli) => {"ansible_loop_var": "item", "changed": false, "checksum": "cb3ef2aa6a3e3002b234f166e739a39b1f3a1bde", "dest": "/usr/sbin/megacli", "gid": 0, "group": "root", "item": "megacli", "mode": "0755", "owner": "root", "path": "/usr/sbin/megacli", "secontext": "system_u:object_r:bin_t:s0", "size": 2716736, "state": "file", "uid": 0} 2021-11-11T20:40:44.007 INFO:teuthology.task.ansible.out: TASK [common : Upload custom netsaint scripts for raid/disk/smart/monitoring to /usr/libexec/.] *** 2021-11-11T20:40:44.009 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=smart.sh) => {"ansible_loop_var": "item", "changed": false, "checksum": "00d0404b545463be4a65f475e0870e07a48f35b6", "dest": "/usr/libexec/smart.sh", "gid": 0, "group": "root", "item": "smart.sh", "mode": "0755", "owner": "root", "path": "/usr/libexec/smart.sh", "secontext": "system_u:object_r:bin_t:s0", "size": 13355, "state": "file", "uid": 0} 2021-11-11T20:40:44.081 INFO:teuthology.task.ansible.out: TASK [common : Upload megacli and cli64 for raid monitoring and smart.pl to /usr/sbin/.] *** 2021-11-11T20:40:44.082 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=cli64) => {"ansible_loop_var": "item", "changed": false, "checksum": "a3b8773b14e8594ffbd9fc156c09436eb0184bc7", "dest": "/usr/sbin/cli64", "gid": 0, "group": "root", "item": "cli64", "mode": "0755", "owner": "root", "path": "/usr/sbin/cli64", "secontext": "system_u:object_r:bin_t:s0", "size": 1952038, "state": "file", "uid": 0} 2021-11-11T20:40:44.413 INFO:teuthology.task.ansible.out: TASK [common : Upload custom netsaint scripts for raid/disk/smart/monitoring to /usr/libexec/.] *** 2021-11-11T20:40:44.415 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=raid.pl) => {"ansible_loop_var": "item", "changed": false, "checksum": "9d9a89805d8a9e4c7281934e887da31f7ac464fb", "dest": "/usr/libexec/raid.pl", "gid": 0, "group": "root", "item": "raid.pl", "mode": "0755", "owner": "root", "path": "/usr/libexec/raid.pl", "secontext": "system_u:object_r:bin_t:s0", "size": 8602, "state": "file", "uid": 0} 2021-11-11T20:40:44.496 INFO:teuthology.task.ansible.out: TASK [common : Upload megacli and cli64 for raid monitoring and smart.pl to /usr/sbin/.] *** 2021-11-11T20:40:44.498 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=nvme) => {"ansible_loop_var": "item", "changed": false, "checksum": "98735dcacd6fec55dd92b67281b478603b7955b1", "dest": "/usr/sbin/nvme", "gid": 0, "group": "root", "item": "nvme", "mode": "0755", "owner": "root", "path": "/usr/sbin/nvme", "secontext": "system_u:object_r:bin_t:s0", "size": 580432, "state": "file", "uid": 0} 2021-11-11T20:40:44.520 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:44 +0000 (0:00:00.985) 0:03:04.776 ***** 2021-11-11T20:40:44.703 INFO:teuthology.task.ansible.out: TASK [common : Create /usr/libexec.] ******************************************* 2021-11-11T20:40:44.705 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/usr/libexec", "secontext": "system_u:object_r:bin_t:s0", "size": 4096, "state": "directory", "uid": 0} 2021-11-11T20:40:44.725 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:44 +0000 (0:00:00.204) 0:03:04.981 ***** 2021-11-11T20:40:44.824 INFO:teuthology.task.ansible.out: TASK [common : Upload custom netsaint scripts for raid/disk/smart/monitoring to /usr/libexec/.] *** 2021-11-11T20:40:44.826 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=diskusage.pl) => {"ansible_loop_var": "item", "changed": false, "checksum": "2fbc67b36d59f66fc2621f3ca2dcf60dea2167a5", "dest": "/usr/libexec/diskusage.pl", "gid": 0, "group": "root", "item": "diskusage.pl", "mode": "0755", "owner": "root", "path": "/usr/libexec/diskusage.pl", "secontext": "system_u:object_r:bin_t:s0", "size": 3612, "state": "file", "uid": 0} 2021-11-11T20:40:44.886 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:44 +0000 (0:00:00.160) 0:03:05.142 ***** 2021-11-11T20:40:44.924 INFO:teuthology.task.ansible.out: TASK [common : Include dnf_system vars] **************************************** 2021-11-11T20:40:44.924 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"nagios_plugins_directory": "/usr/lib64/nagios/plugins", "nrpe_group": "nrpe", "nrpe_packages": ["nagios-common", "nrpe", "nagios-plugins", "nagios-plugins-load"], "nrpe_service_name": "nrpe", "nrpe_user": "nrpe"}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/common/vars/dnf_systems.yml"], "changed": false} 2021-11-11T20:40:44.945 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:44 +0000 (0:00:00.058) 0:03:05.201 ***** 2021-11-11T20:40:45.137 INFO:teuthology.task.ansible.out: TASK [common : Check for epel] ************************************************* 2021-11-11T20:40:45.139 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": true} 2021-11-11T20:40:45.159 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:45 +0000 (0:00:00.214) 0:03:05.415 ***** 2021-11-11T20:40:45.195 INFO:teuthology.task.ansible.out: TASK [common : Install nrpe without epel] ************************************** 2021-11-11T20:40:45.197 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-common-4.0.8-2.el7.x86_64.rpm) => {"ansible_loop_var": "item", "changed": false, "item": "http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-common-4.0.8-2.el7.x86_64.rpm", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:45.207 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nrpe-2.15-7.el7.x86_64.rpm) => {"ansible_loop_var": "item", "changed": false, "item": "http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nrpe-2.15-7.el7.x86_64.rpm", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:45.217 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-plugins-2.0.3-3.el7.x86_64.rpm) => {"ansible_loop_var": "item", "changed": false, "item": "http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-plugins-2.0.3-3.el7.x86_64.rpm", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:45.219 INFO:teuthology.task.ansible.out: TASK [common : Upload custom netsaint scripts for raid/disk/smart/monitoring to /usr/libexec/.] *** ok: [smithi198.front.sepia.ceph.com] => (item=smart.sh) => {"ansible_loop_var": "item", "changed": false, "checksum": "00d0404b545463be4a65f475e0870e07a48f35b6", "dest": "/usr/libexec/smart.sh", "gid": 0, "group": "root", "item": "smart.sh", "mode": "0755", "owner": "root", "path": "/usr/libexec/smart.sh", "secontext": "system_u:object_r:bin_t:s0", "size": 13355, "state": "file", "uid": 0} 2021-11-11T20:40:45.219 INFO:teuthology.task.ansible.out: TASK [common : Install nrpe without epel] ************************************** skipping: [smithi096.front.sepia.ceph.com] => (item=http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-plugins-load-2.0.3-3.el7.x86_64.rpm) => {"ansible_loop_var": "item", "changed": false, "item": "http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-plugins-load-2.0.3-3.el7.x86_64.rpm", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:45.242 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:45 +0000 (0:00:00.083) 0:03:05.499 ***** 2021-11-11T20:40:45.632 INFO:teuthology.task.ansible.out: TASK [common : Upload custom netsaint scripts for raid/disk/smart/monitoring to /usr/libexec/.] *** 2021-11-11T20:40:45.634 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=raid.pl) => {"ansible_loop_var": "item", "changed": false, "checksum": "9d9a89805d8a9e4c7281934e887da31f7ac464fb", "dest": "/usr/libexec/raid.pl", "gid": 0, "group": "root", "item": "raid.pl", "mode": "0755", "owner": "root", "path": "/usr/libexec/raid.pl", "secontext": "system_u:object_r:bin_t:s0", "size": 8602, "state": "file", "uid": 0} 2021-11-11T20:40:46.026 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=diskusage.pl) => {"ansible_loop_var": "item", "changed": false, "checksum": "2fbc67b36d59f66fc2621f3ca2dcf60dea2167a5", "dest": "/usr/libexec/diskusage.pl", "gid": 0, "group": "root", "item": "diskusage.pl", "mode": "0755", "owner": "root", "path": "/usr/libexec/diskusage.pl", "secontext": "system_u:object_r:bin_t:s0", "size": 3612, "state": "file", "uid": 0} 2021-11-11T20:40:46.051 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:46 +0000 (0:00:00.808) 0:03:06.307 ***** 2021-11-11T20:40:46.089 INFO:teuthology.task.ansible.out: TASK [common : Include dnf_system vars] **************************************** 2021-11-11T20:40:46.091 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"nagios_plugins_directory": "/usr/lib64/nagios/plugins", "nrpe_group": "nrpe", "nrpe_packages": ["nagios-common", "nrpe", "nagios-plugins", "nagios-plugins-load"], "nrpe_service_name": "nrpe", "nrpe_user": "nrpe"}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/common/vars/dnf_systems.yml"], "changed": false} 2021-11-11T20:40:46.110 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:46 +0000 (0:00:00.059) 0:03:06.366 ***** 2021-11-11T20:40:46.303 INFO:teuthology.task.ansible.out: TASK [common : Check for epel] ************************************************* 2021-11-11T20:40:46.305 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": true} 2021-11-11T20:40:46.325 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:46 +0000 (0:00:00.214) 0:03:06.581 ***** 2021-11-11T20:40:46.372 INFO:teuthology.task.ansible.out: TASK [common : Install nrpe without epel] ************************************** 2021-11-11T20:40:46.375 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-common-4.0.8-2.el7.x86_64.rpm) => {"ansible_loop_var": "item", "changed": false, "item": "http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-common-4.0.8-2.el7.x86_64.rpm", "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item=http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nrpe-2.15-7.el7.x86_64.rpm) => {"ansible_loop_var": "item", "changed": false, "item": "http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nrpe-2.15-7.el7.x86_64.rpm", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:46.384 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-plugins-2.0.3-3.el7.x86_64.rpm) => {"ansible_loop_var": "item", "changed": false, "item": "http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-plugins-2.0.3-3.el7.x86_64.rpm", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:46.395 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-plugins-load-2.0.3-3.el7.x86_64.rpm) => {"ansible_loop_var": "item", "changed": false, "item": "http://apt-mirror.front.sepia.ceph.com/lab-extras/rhel7/x86_64/nagios-plugins-load-2.0.3-3.el7.x86_64.rpm", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:46.420 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:46 +0000 (0:00:00.094) 0:03:06.676 ***** 2021-11-11T20:40:48.225 INFO:teuthology.task.ansible.out: TASK [common : Install nrpe package and dependencies (RHEL/CentOS)] ************ 2021-11-11T20:40:48.227 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:40:48.247 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:48 +0000 (0:00:01.827) 0:03:08.503 ***** 2021-11-11T20:40:48.274 INFO:teuthology.task.ansible.out: TASK [common : Install nrpe package and dependencies (non-RHEL/CentOS)] ******** 2021-11-11T20:40:48.276 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:48.295 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:48 +0000 (0:00:00.047) 0:03:08.551 ***** 2021-11-11T20:40:48.734 INFO:teuthology.task.ansible.out: TASK [common : Upload nagios sudoers.d for raid utilities.] ******************** 2021-11-11T20:40:48.736 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "b423b985ccbb69b65a711d372645a5f704b30e8b", "dest": "/etc/sudoers.d/90-nagios", "gid": 0, "group": "root", "mode": "0440", "owner": "root", "path": "/etc/sudoers.d/90-nagios", "secontext": "system_u:object_r:etc_t:s0", "size": 178, "state": "file", "uid": 0} 2021-11-11T20:40:48.755 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:48 +0000 (0:00:00.460) 0:03:09.011 ***** 2021-11-11T20:40:49.185 INFO:teuthology.task.ansible.out: TASK [common : Upload nagios check_mem script] ********************************* 2021-11-11T20:40:49.187 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "39df629b10ded370443e2e4c84d690332c95104d", "dest": "/usr/lib64/nagios/plugins/check_mem.sh", "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/usr/lib64/nagios/plugins/check_mem.sh", "secontext": "system_u:object_r:nagios_unconfined_plugin_exec_t:s0", "size": 1964, "state": "file", "uid": 0} 2021-11-11T20:40:49.207 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:49 +0000 (0:00:00.452) 0:03:09.463 ***** 2021-11-11T20:40:49.244 INFO:teuthology.task.ansible.out: TASK [common : Configure nagios nrpe settings (Ubuntu)] ************************ 2021-11-11T20:40:49.246 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=DAEMON_OPTS) => {"ansible_loop_var": "item", "changed": false, "item": "DAEMON_OPTS", "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item=NRPE_OPTS) => {"ansible_loop_var": "item", "changed": false, "item": "NRPE_OPTS", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:49.267 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:49 +0000 (0:00:00.059) 0:03:09.523 ***** 2021-11-11T20:40:49.401 INFO:teuthology.task.ansible.out: TASK [common : Install nrpe package and dependencies (RHEL/CentOS)] ************ 2021-11-11T20:40:49.403 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:40:49.428 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:49 +0000 (0:00:00.160) 0:03:09.684 ***** 2021-11-11T20:40:49.456 INFO:teuthology.task.ansible.out: TASK [common : Install nrpe package and dependencies (non-RHEL/CentOS)] ******** 2021-11-11T20:40:49.458 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:49.479 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:49 +0000 (0:00:00.050) 0:03:09.734 ***** 2021-11-11T20:40:49.486 INFO:teuthology.task.ansible.out: TASK [common : Configure nagios nrpe settings (RHEL/CentOS)] ******************* 2021-11-11T20:40:49.487 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"backup": "", "changed": false, "msg": ""} 2021-11-11T20:40:49.510 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:49 +0000 (0:00:00.031) 0:03:09.766 ***** 2021-11-11T20:40:49.734 INFO:teuthology.task.ansible.out: TASK [common : Check firewalld status] ***************************************** 2021-11-11T20:40:49.736 INFO:teuthology.task.ansible.out:fatal: [smithi096.front.sepia.ceph.com]: FAILED! => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": true} ...ignoring 2021-11-11T20:40:49.757 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:49 +0000 (0:00:00.246) 0:03:10.013 ***** 2021-11-11T20:40:49.785 INFO:teuthology.task.ansible.out: TASK [common : Open nrpe port if firewalld enabled] **************************** 2021-11-11T20:40:49.787 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:49.807 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:49 +0000 (0:00:00.049) 0:03:10.063 ***** 2021-11-11T20:40:49.927 INFO:teuthology.task.ansible.out: TASK [common : Upload nagios sudoers.d for raid utilities.] ******************** 2021-11-11T20:40:49.928 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "b423b985ccbb69b65a711d372645a5f704b30e8b", "dest": "/etc/sudoers.d/90-nagios", "gid": 0, "group": "root", "mode": "0440", "owner": "root", "path": "/etc/sudoers.d/90-nagios", "secontext": "system_u:object_r:etc_t:s0", "size": 178, "state": "file", "uid": 0} 2021-11-11T20:40:49.950 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:49 +0000 (0:00:00.143) 0:03:10.206 ***** 2021-11-11T20:40:50.247 INFO:teuthology.task.ansible.out: TASK [common : Upload nagios nrpe config.] ************************************* 2021-11-11T20:40:50.249 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "aa9150d4582f919d7a8504c01c8d86f05e3eeb5d", "dest": "/etc/nagios/nrpe.cfg", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/nagios/nrpe.cfg", "secontext": "system_u:object_r:nrpe_etc_t:s0", "size": 1184, "state": "file", "uid": 0} 2021-11-11T20:40:50.271 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:50 +0000 (0:00:00.320) 0:03:10.527 ***** 2021-11-11T20:40:50.391 INFO:teuthology.task.ansible.out: TASK [common : Upload nagios check_mem script] ********************************* 2021-11-11T20:40:50.394 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "39df629b10ded370443e2e4c84d690332c95104d", "dest": "/usr/lib64/nagios/plugins/check_mem.sh", "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/usr/lib64/nagios/plugins/check_mem.sh", "secontext": "system_u:object_r:nagios_unconfined_plugin_exec_t:s0", "size": 1964, "state": "file", "uid": 0} 2021-11-11T20:40:50.414 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:50 +0000 (0:00:00.143) 0:03:10.670 ***** 2021-11-11T20:40:50.452 INFO:teuthology.task.ansible.out: TASK [common : Configure nagios nrpe settings (Ubuntu)] ************************ 2021-11-11T20:40:50.454 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=DAEMON_OPTS) => {"ansible_loop_var": "item", "changed": false, "item": "DAEMON_OPTS", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:50.463 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=NRPE_OPTS) => {"ansible_loop_var": "item", "changed": false, "item": "NRPE_OPTS", "skip_reason": "Conditional result was False"} 2021-11-11T20:40:50.486 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:50 +0000 (0:00:00.071) 0:03:10.742 ***** 2021-11-11T20:40:50.679 INFO:teuthology.task.ansible.out: TASK [common : Configure nagios nrpe settings (RHEL/CentOS)] ******************* 2021-11-11T20:40:50.680 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"backup": "", "changed": false, "msg": ""} 2021-11-11T20:40:50.701 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:50 +0000 (0:00:00.215) 0:03:10.957 ***** 2021-11-11T20:40:50.904 INFO:teuthology.task.ansible.out: TASK [common : Make sure nagios nrpe service is running.] ********************** 2021-11-11T20:40:50.906 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "enabled": true, "name": "nrpe", "state": "started", "status": {"ActiveEnterTimestamp": "Thu 2021-11-11 20:36:27 UTC", "ActiveEnterTimestampMonotonic": "22332487", "ActiveExitTimestampMonotonic": "0", "ActiveState": "active", "After": "systemd-journald.socket local-fs.target network-online.target nss-lookup.target time-sync.target system.slice basic.target sysinit.target -.mount var-run.mount tmp.mount systemd-tmpfiles-setup.service", "AllowIsolate": "no", "AllowedCPUs": "", "AllowedMemoryNodes": "", "AmbientCapabilities": "", "AssertResult": "yes", "AssertTimestamp": "Thu 2021-11-11 20:36:27 UTC", "AssertTimestampMonotonic": "22309531", "Before": "xdm.service shutdown.target multi-user.target plymouth-quit.service getty@tty1.service", "BlockIOAccounting": "no", "BlockIOWeight": "[not set]", "CPUAccounting": "no", "CPUAffinity": "", "CPUAffinityFromNUMA": "no", "CPUQuotaPerSecUSec": "infinity", "CPUQuotaPeriodUSec": "infinity", "CPUSchedulingPolicy": "0", "CPUSchedulingPriority": "0", "CPUSchedulingResetOnFork": "no", "CPUShares": "[not set]", "CPUUsageNSec": "[not set]", "CPUWeight": "[not set]", "CacheDirectoryMode": "0755", "CanFreeze": "yes", "CanIsolate": "no", "CanReload": "yes", "CanStart": "yes", "CanStop": "yes", "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon", "CollectMode": "inactive", "ConditionResult": "yes", "ConditionTimestamp": "Thu 2021-11-11 20:36:27 UTC", "ConditionT 2021-11-11T20:40:50.907 INFO:teuthology.task.ansible.out:imestampMonotonic": "22309531", "ConfigurationDirectoryMode": "0755", "Conflicts": "nrpe.socket shutdown.target", "ControlGroup": "/system.slice/nrpe.service", "ControlPID": "0", "DefaultDependencies": "yes", "DefaultMemoryLow": "0", "DefaultMemoryMin": "0", "Delegate": "no", "Description": "Nagios Remote Plugin Executor", "DevicePolicy": "auto", "Documentation": "http://www.nagios.org/documentation", "DynamicUser": "no", "EffectiveCPUs": "", "EffectiveMemoryNodes": "", "EnvironmentFiles": "/etc/sysconfig/nrpe (ignore_errors=yes)", "ExecMainCode": "0", "ExecMainExitTimestampMonotonic": "0", "ExecMainPID": "1233", "ExecMainStartTimestamp": "Thu 2021-11-11 20:36:27 UTC", "ExecMainStartTimestampMonotonic": "22332459", "ExecMainStatus": "0", "ExecReload": "{ path=/bin/kill ; argv[]=/bin/kill -HUP $MAINPID ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "ExecStart": "{ path=/usr/sbin/nrpe ; argv[]=/usr/sbin/nrpe -c /etc/nagios/nrpe.cfg -f $NRPE_SSL_OPT ; ignore_errors=no ; start_time=[Thu 2021-11-11 20:36:27 UTC] ; stop_time=[n/a] ; pid=1233 ; code=(null) ; status=0/0 }", "ExecStopPost": "{ path=/bin/rm ; argv[]=/bin/rm -f /run/nrpe/nrpe.pid ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "FailureAction": "none", "FileDescriptorStoreMax": "0", "FragmentPath": "/usr/lib/systemd/system/nrpe.service", "FreezerState": "running", "GID": "985", "Group": "nrpe", "GuessMainPID": "yes", "IOAccounting": "no", "IOSchedulingClass": "0", "IOSchedulingPriority": "0", "IOWeight": "[not set]", "IPAccounting": "no", "IPEgressBytes": "18446744073709551615", "IPEgressPackets": "18446744073709551615", "IPIngressBytes": "18446744073709551615", "IPIngressPackets": "18446744073709551615", "Id": "nrpe.service", "IgnoreOnIsolate": "no", "IgnoreSIGPIPE": "yes", "InactiveEnterTimestampMonotonic": "0", "InactiveExitTimestamp": "Thu 2021-11-11 20:36:27 UTC", "InactiveExitTimestampMonotonic": "22332487", "In 2021-11-11T20:40:50.909 INFO:teuthology.task.ansible.out:vocationID": "38ecbbfdd4ba485683e344b51db4fc6e", "JobRunningTimeoutUSec": "infinity", "JobTimeoutAction": "none", "JobTimeoutUSec": "infinity", "KeyringMode": "private", "KillMode": "control-group", "KillSignal": "15", "LimitAS": "infinity", "LimitASSoft": "infinity", "LimitCORE": "infinity", "LimitCORESoft": "infinity", "LimitCPU": "infinity", "LimitCPUSoft": "infinity", "LimitDATA": "infinity", "LimitDATASoft": "infinity", "LimitFSIZE": "infinity", "LimitFSIZESoft": "infinity", "LimitLOCKS": "infinity", "LimitLOCKSSoft": "infinity", "LimitMEMLOCK": "65536", "LimitMEMLOCKSoft": "65536", "LimitMSGQUEUE": "819200", "LimitMSGQUEUESoft": "819200", "LimitNICE": "0", "LimitNICESoft": "0", "LimitNOFILE": "262144", "LimitNOFILESoft": "1024", "LimitNPROC": "127240", "LimitNPROCSoft": "127240", "LimitRSS": "infinity", "LimitRSSSoft": "infinity", "LimitRTPRIO": "0", "LimitRTPRIOSoft": "0", "LimitRTTIME": "infinity", "LimitRTTIMESoft": "infinity", "LimitSIGPENDING": "127240", "LimitSIGPENDINGSoft": "127240", "LimitSTACK": "infinity", "LimitSTACKSoft": "8388608", "LoadState": "loaded", "LockPersonality": "no", "LogLevelMax": "-1", "LogRateLimitBurst": "0", "LogRateLimitIntervalUSec": "0", "LogsDirectoryMode": "0755", "MainPID": "1233", "MemoryAccounting": "yes", "MemoryCurrent": "671744", "MemoryDenyWriteExecute": "no", "MemoryHigh": "infinity", "MemoryLimit": "infinity", "MemoryLow": "0", "MemoryMax": "infinity", "MemoryMin": "0", "MemorySwapMax": "infinity", "MountAPIVFS": "no", "MountFlags": "", "NFileDescriptorStore": "0", "NRestarts": "0", "NUMAMask": "", "NUMAPolicy": "n/a", "Names": "nrpe.service", "NeedDaemonReload": "no", "Nice": "0", "NoNewPrivileges": "no", "NonBlocking": "no", "NotifyAccess": "none", "OOMScoreAdjust": "-500", "OnFailureJobMode": "replace", "PIDFile": "/run/nrpe/nrpe.pid", "PermissionsStartOnly": "no", "Perpetual": "no", "PrivateDevices": "no", "PrivateMounts": "no", "PrivateNetwork": "no", "PrivateTmp": "yes", "PrivateUsers": "no", "ProtectControlGr 2021-11-11T20:40:50.910 INFO:teuthology.task.ansible.out:oups": "no", "ProtectHome": "no", "ProtectKernelModules": "no", "ProtectKernelTunables": "no", "ProtectSystem": "no", "RefuseManualStart": "no", "RefuseManualStop": "no", "RemainAfterExit": "no", "RemoveIPC": "no", "Requires": "system.slice -.mount sysinit.target", "RequiresMountsFor": "/var/tmp /run/nrpe", "Restart": "on-abort", "RestartUSec": "100ms", "RestrictNamespaces": "no", "RestrictRealtime": "no", "RestrictSUIDSGID": "no", "Result": "success", "RootDirectoryStartOnly": "no", "RuntimeDirectory": "nrpe", "RuntimeDirectoryMode": "0755", "RuntimeDirectoryPreserve": "no", "RuntimeMaxUSec": "infinity", "SameProcessGroup": "no", "SecureBits": "0", "SendSIGHUP": "no", "SendSIGKILL": "yes", "Slice": "system.slice", "StandardError": "inherit", "StandardInput": "null", "StandardInputData": "", "StandardOutput": "journal", "StartLimitAction": "none", "StartLimitBurst": "5", "StartLimitIntervalUSec": "10s", "StartupBlockIOWeight": "[not set]", "StartupCPUShares": "[not set]", "StartupCPUWeight": "[not set]", "StartupIOWeight": "[not set]", "StateChangeTimestamp": "Thu 2021-11-11 20:36:27 UTC", "StateChangeTimestampMonotonic": "22332487", "StateDirectoryMode": "0755", "StatusErrno": "0", "StopWhenUnneeded": "no", "SubState": "running", "SuccessAction": "none", "SyslogFacility": "3", "SyslogLevel": "6", "SyslogLevelPrefix": "yes", "SyslogPriority": "30", "SystemCallErrorNumber": "0", "TTYReset": "no", "TTYVHangup": "no", "TTYVTDisallocate": "no", "TasksAccounting": "yes", "TasksCurrent": "1", "TasksMax": "203584", "TimeoutStartUSec": "1min 30s", "TimeoutStopUSec": "1min", "TimerSlackNSec": "50000", "Transient": "no", "Type": "simple", "UID": "989", "UMask": "0022", "UnitFilePreset": "disabled", "UnitFileState": "enabled", "User": "nrpe", "UtmpMode": "init", "WantedBy": "multi-user.target", "WatchdogTimestamp": "Thu 2021-11-11 20:36:27 UTC", "WatchdogTimestampMonotonic": "22332486", "WatchdogUSec": "0"}} 2021-11-11T20:40:50.930 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:50 +0000 (0:00:00.228) 0:03:11.186 ***** 2021-11-11T20:40:50.936 INFO:teuthology.task.ansible.out: TASK [common : Check firewalld status] ***************************************** 2021-11-11T20:40:50.938 INFO:teuthology.task.ansible.out:fatal: [smithi198.front.sepia.ceph.com]: FAILED! => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": true} ...ignoring 2021-11-11T20:40:50.961 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:50 +0000 (0:00:00.030) 0:03:11.217 ***** 2021-11-11T20:40:50.987 INFO:teuthology.task.ansible.out: TASK [common : Open nrpe port if firewalld enabled] **************************** 2021-11-11T20:40:50.989 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:40:51.013 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:51 +0000 (0:00:00.052) 0:03:11.269 ***** 2021-11-11T20:40:51.112 INFO:teuthology.task.ansible.out: TASK [common : Get SELinux status] ********************************************* 2021-11-11T20:40:51.114 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": ["getenforce"], "delta": "0:00:00.001801", "end": "2021-11-11 20:40:51.093376", "rc": 0, "start": "2021-11-11 20:40:51.091575", "stderr": "", "stderr_lines": [], "stdout": "Permissive", "stdout_lines": ["Permissive"]} 2021-11-11T20:40:51.138 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:51 +0000 (0:00:00.125) 0:03:11.394 ***** 2021-11-11T20:40:51.468 INFO:teuthology.task.ansible.out: TASK [common : Upload nagios nrpe config.] ************************************* 2021-11-11T20:40:51.470 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "aa9150d4582f919d7a8504c01c8d86f05e3eeb5d", "dest": "/etc/nagios/nrpe.cfg", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/nagios/nrpe.cfg", "secontext": "system_u:object_r:nrpe_etc_t:s0", "size": 1184, "state": "file", "uid": 0} 2021-11-11T20:40:51.491 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:51 +0000 (0:00:00.353) 0:03:11.747 ***** 2021-11-11T20:40:51.788 INFO:teuthology.task.ansible.out: TASK [common : Make sure nagios nrpe service is running.] ********************** 2021-11-11T20:40:51.789 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "enabled": true, "name": "nrpe", "state": "started", "status": {"ActiveEnterTimestamp": "Thu 2021-11-11 20:36:20 UTC", "ActiveEnterTimestampMonotonic": "22032330", "ActiveExitTimestampMonotonic": "0", "ActiveState": "active", "After": "local-fs.target tmp.mount var-run.mount basic.target systemd-tmpfiles-setup.service nss-lookup.target system.slice -.mount systemd-journald.socket time-sync.target sysinit.target network-online.target", "AllowIsolate": "no", "AllowedCPUs": "", "AllowedMemoryNodes": "", "AmbientCapabilities": "", "AssertResult": "yes", "AssertTimestamp": "Thu 2021-11-11 20:36:20 UTC", "AssertTimestampMonotonic": "22024160", "Before": "getty@tty1.service shutdown.target xdm.service plymouth-quit.service multi-user.target", "BlockIOAccounting": "no", "BlockIOWeight": "[not set]", "CPUAccounting": "no", "CPUAffinity": "", "CPUAffinityFromNUMA": "no", "CPUQuotaPerSecUSec": "infinity", "CPUQuotaPeriodUSec": "infinity", "CPUSchedulingPolicy": "0", "CPUSchedulingPriority": "0", "CPUSchedulingResetOnFork": "no", "CPUShares": "[not set]", "CPUUsageNSec": "[not set]", "CPUWeight": "[not set]", "CacheDirectoryMode": "0755", "CanFreeze": "yes", "CanIsolate": "no", "CanReload": "yes", "CanStart": "yes", "CanStop": "yes", "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon", "CollectMode": "inactive", "ConditionResult": "yes", "ConditionTimestamp": "Thu 2021-11-11 20:36:20 UTC", "ConditionT 2021-11-11T20:40:51.791 INFO:teuthology.task.ansible.out:imestampMonotonic": "22024160", "ConfigurationDirectoryMode": "0755", "Conflicts": "nrpe.socket shutdown.target", "ControlGroup": "/system.slice/nrpe.service", "ControlPID": "0", "DefaultDependencies": "yes", "DefaultMemoryLow": "0", "DefaultMemoryMin": "0", "Delegate": "no", "Description": "Nagios Remote Plugin Executor", "DevicePolicy": "auto", "Documentation": "http://www.nagios.org/documentation", "DynamicUser": "no", "EffectiveCPUs": "", "EffectiveMemoryNodes": "", "EnvironmentFiles": "/etc/sysconfig/nrpe (ignore_errors=yes)", "ExecMainCode": "0", "ExecMainExitTimestampMonotonic": "0", "ExecMainPID": "1233", "ExecMainStartTimestamp": "Thu 2021-11-11 20:36:20 UTC", "ExecMainStartTimestampMonotonic": "22032282", "ExecMainStatus": "0", "ExecReload": "{ path=/bin/kill ; argv[]=/bin/kill -HUP $MAINPID ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "ExecStart": "{ path=/usr/sbin/nrpe ; argv[]=/usr/sbin/nrpe -c /etc/nagios/nrpe.cfg -f $NRPE_SSL_OPT ; ignore_errors=no ; start_time=[Thu 2021-11-11 20:36:20 UTC] ; stop_time=[n/a] ; pid=1233 ; code=(null) ; status=0/0 }", "ExecStopPost": "{ path=/bin/rm ; argv[]=/bin/rm -f /run/nrpe/nrpe.pid ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "FailureAction": "none", "FileDescriptorStoreMax": "0", "FragmentPath": "/usr/lib/systemd/system/nrpe.service", "FreezerState": "running", "GID": "985", "Group": "nrpe", "GuessMainPID": "yes", "IOAccounting": "no", "IOSchedulingClass": "0", "IOSchedulingPriority": "0", "IOWeight": "[not set]", "IPAccounting": "no", "IPEgressBytes": "18446744073709551615", "IPEgressPackets": "18446744073709551615", "IPIngressBytes": "18446744073709551615", "IPIngressPackets": "18446744073709551615", "Id": "nrpe.service", "IgnoreOnIsolate": "no", "IgnoreSIGPIPE": "yes", "InactiveEnterTimestampMonotonic": "0", "InactiveExitTimestamp": "Thu 2021-11-11 20:36:20 UTC", "InactiveExitTimestampMonotonic": "22032330", "In 2021-11-11T20:40:51.792 INFO:teuthology.task.ansible.out:vocationID": "442114ce345f44b9b35674c85e521199", "JobRunningTimeoutUSec": "infinity", "JobTimeoutAction": "none", "JobTimeoutUSec": "infinity", "KeyringMode": "private", "KillMode": "control-group", "KillSignal": "15", "LimitAS": "infinity", "LimitASSoft": "infinity", "LimitCORE": "infinity", "LimitCORESoft": "infinity", "LimitCPU": "infinity", "LimitCPUSoft": "infinity", "LimitDATA": "infinity", "LimitDATASoft": "infinity", "LimitFSIZE": "infinity", "LimitFSIZESoft": "infinity", "LimitLOCKS": "infinity", "LimitLOCKSSoft": "infinity", "LimitMEMLOCK": "65536", "LimitMEMLOCKSoft": "65536", "LimitMSGQUEUE": "819200", "LimitMSGQUEUESoft": "819200", "LimitNICE": "0", "LimitNICESoft": "0", "LimitNOFILE": "262144", "LimitNOFILESoft": "1024", "LimitNPROC": "127240", "LimitNPROCSoft": "127240", "LimitRSS": "infinity", "LimitRSSSoft": "infinity", "LimitRTPRIO": "0", "LimitRTPRIOSoft": "0", "LimitRTTIME": "infinity", "LimitRTTIMESoft": "infinity", "LimitSIGPENDING": "127240", "LimitSIGPENDINGSoft": "127240", "LimitSTACK": "infinity", "LimitSTACKSoft": "8388608", "LoadState": "loaded", "LockPersonality": "no", "LogLevelMax": "-1", "LogRateLimitBurst": "0", "LogRateLimitIntervalUSec": "0", "LogsDirectoryMode": "0755", "MainPID": "1233", "MemoryAccounting": "yes", "MemoryCurrent": "679936", "MemoryDenyWriteExecute": "no", "MemoryHigh": "infinity", "MemoryLimit": "infinity", "MemoryLow": "0", "MemoryMax": "infinity", "MemoryMin": "0", "MemorySwapMax": "infinity", "MountAPIVFS": "no", "MountFlags": "", "NFileDescriptorStore": "0", "NRestarts": "0", "NUMAMask": "", "NUMAPolicy": "n/a", "Names": "nrpe.service", "NeedDaemonReload": "no", "Nice": "0", "NoNewPrivileges": "no", "NonBlocking": "no", "NotifyAccess": "none", "OOMScoreAdjust": "-500", "OnFailureJobMode": "replace", "PIDFile": "/run/nrpe/nrpe.pid", "PermissionsStartOnly": "no", "Perpetual": "no", "PrivateDevices": "no", "PrivateMounts": "no", "PrivateNetwork": "no", "PrivateTmp": "yes", "PrivateUsers": "no", "ProtectControlGr 2021-11-11T20:40:51.794 INFO:teuthology.task.ansible.out:oups": "no", "ProtectHome": "no", "ProtectKernelModules": "no", "ProtectKernelTunables": "no", "ProtectSystem": "no", "RefuseManualStart": "no", "RefuseManualStop": "no", "RemainAfterExit": "no", "RemoveIPC": "no", "Requires": "-.mount sysinit.target system.slice", "RequiresMountsFor": "/var/tmp /run/nrpe", "Restart": "on-abort", "RestartUSec": "100ms", "RestrictNamespaces": "no", "RestrictRealtime": "no", "RestrictSUIDSGID": "no", "Result": "success", "RootDirectoryStartOnly": "no", "RuntimeDirectory": "nrpe", "RuntimeDirectoryMode": "0755", "RuntimeDirectoryPreserve": "no", "RuntimeMaxUSec": "infinity", "SameProcessGroup": "no", "SecureBits": "0", "SendSIGHUP": "no", "SendSIGKILL": "yes", "Slice": "system.slice", "StandardError": "inherit", "StandardInput": "null", "StandardInputData": "", "StandardOutput": "journal", "StartLimitAction": "none", "StartLimitBurst": "5", "StartLimitIntervalUSec": "10s", "StartupBlockIOWeight": "[not set]", "StartupCPUShares": "[not set]", "StartupCPUWeight": "[not set]", "StartupIOWeight": "[not set]", "StateChangeTimestamp": "Thu 2021-11-11 20:36:20 UTC", "StateChangeTimestampMonotonic": "22032330", "StateDirectoryMode": "0755", "StatusErrno": "0", "StopWhenUnneeded": "no", "SubState": "running", "SuccessAction": "none", "SyslogFacility": "3", "SyslogLevel": "6", "SyslogLevelPrefix": "yes", "SyslogPriority": "30", "SystemCallErrorNumber": "0", "TTYReset": "no", "TTYVHangup": "no", "TTYVTDisallocate": "no", "TasksAccounting": "yes", "TasksCurrent": "1", "TasksMax": "203584", "TimeoutStartUSec": "1min 30s", "TimeoutStopUSec": "1min", "TimerSlackNSec": "50000", "Transient": "no", "Type": "simple", "UID": "989", "UMask": "0022", "UnitFilePreset": "disabled", "UnitFileState": "enabled", "User": "nrpe", "UtmpMode": "init", "WantedBy": "multi-user.target", "WatchdogTimestamp": "Thu 2021-11-11 20:36:20 UTC", "WatchdogTimestampMonotonic": "22032329", "WatchdogUSec": "0"}} 2021-11-11T20:40:51.811 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:51 +0000 (0:00:00.319) 0:03:12.067 ***** 2021-11-11T20:40:52.004 INFO:teuthology.task.ansible.out: TASK [common : Get SELinux status] ********************************************* 2021-11-11T20:40:52.005 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": ["getenforce"], "delta": "0:00:00.001818", "end": "2021-11-11 20:40:51.986427", "rc": 0, "start": "2021-11-11 20:40:51.984609", "stderr": "", "stderr_lines": [], "stdout": "Permissive", "stdout_lines": ["Permissive"]} 2021-11-11T20:40:52.026 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:52 +0000 (0:00:00.214) 0:03:12.282 ***** 2021-11-11T20:40:54.087 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Install SELinux dependencies] **************************** 2021-11-11T20:40:54.089 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:40:54.111 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:54 +0000 (0:00:02.084) 0:03:14.367 ***** 2021-11-11T20:40:55.044 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:40:55.066 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:40:55 +0000 (0:00:00.954) 0:03:15.322 ***** 2021-11-11T20:41:47.931 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Ensure SELinux policy is up to date] ********************* 2021-11-11T20:41:47.933 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "msg": "", "rc": 0, "results": ["Installed: selinux-policy-targeted-3.14.3-67.el8_4.2.noarch", "Installed: selinux-policy-3.14.3-67.el8_4.2.noarch", "Removed: selinux-policy-targeted-3.14.3-67.el8.noarch", "Removed: selinux-policy-3.14.3-67.el8.noarch"]} 2021-11-11T20:41:47.957 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:41:47 +0000 (0:00:52.891) 0:04:08.213 ***** 2021-11-11T20:41:48.652 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Set SELinux boolean nagios_run_sudo true] **************** 2021-11-11T20:41:48.654 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "name": "nagios_run_sudo", "persistent": true, "state": true} 2021-11-11T20:41:48.676 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:41:48 +0000 (0:00:00.717) 0:04:08.931 ***** 2021-11-11T20:41:48.934 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Ensure SELinux policy is up to date] ********************* 2021-11-11T20:41:48.936 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "msg": "", "rc": 0, "results": ["Installed: selinux-policy-targeted-3.14.3-67.el8_4.2.noarch", "Installed: selinux-policy-3.14.3-67.el8_4.2.noarch", "Removed: selinux-policy-targeted-3.14.3-67.el8.noarch", "Removed: selinux-policy-3.14.3-67.el8.noarch"]} 2021-11-11T20:41:48.958 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:41:48 +0000 (0:00:00.282) 0:04:09.214 ***** 2021-11-11T20:41:49.401 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Set SELinux boolean nagios_run_sudo true] **************** 2021-11-11T20:41:49.403 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "name": "nagios_run_sudo", "persistent": true, "state": true} 2021-11-11T20:41:49.431 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:41:49 +0000 (0:00:00.473) 0:04:09.687 ***** 2021-11-11T20:42:04.004 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Remove SELinux policy package] *************************** 2021-11-11T20:42:04.006 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": ["semodule", "-r", "nrpe"], "delta": "0:00:15.132796", "end": "2021-11-11 20:42:03.984347", "failed_when_result": false, "rc": 0, "start": "2021-11-11 20:41:48.851551", "stderr": "", "stderr_lines": [], "stdout": "libsemanage.semanage_direct_remove_key: Removing last nrpe module (no other nrpe module exists at another priority).", "stdout_lines": ["libsemanage.semanage_direct_remove_key: Removing last nrpe module (no other nrpe module exists at another priority)."]} 2021-11-11T20:42:04.027 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:04 +0000 (0:00:14.595) 0:04:24.283 ***** 2021-11-11T20:42:04.483 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Copy SELinux type enforcement file] ********************** 2021-11-11T20:42:04.486 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "99d9b2ea41fb2d6613898129718064833d2de428", "dest": "/tmp/nrpe.te", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/tmp/nrpe.te", "secontext": "unconfined_u:object_r:user_home_t:s0", "size": 934, "state": "file", "uid": 0} 2021-11-11T20:42:04.516 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:04 +0000 (0:00:00.488) 0:04:24.771 ***** 2021-11-11T20:42:04.715 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Remove SELinux policy package] *************************** 2021-11-11T20:42:04.717 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": ["semodule", "-r", "nrpe"], "delta": "0:00:15.093606", "end": "2021-11-11 20:42:04.695461", "failed_when_result": false, "rc": 0, "start": "2021-11-11 20:41:49.601855", "stderr": "", "stderr_lines": [], "stdout": "libsemanage.semanage_direct_remove_key: Removing last nrpe module (no other nrpe module exists at another priority).", "stdout_lines": ["libsemanage.semanage_direct_remove_key: Removing last nrpe module (no other nrpe module exists at another priority)."]} 2021-11-11T20:42:04.738 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:04 +0000 (0:00:00.222) 0:04:24.994 ***** 2021-11-11T20:42:04.746 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Compile SELinux module file] ***************************** 2021-11-11T20:42:04.748 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": ["checkmodule", "-M", "-m", "-o", "/tmp/nrpe.mod", "/tmp/nrpe.te"], "delta": "0:00:00.018483", "end": "2021-11-11 20:42:04.704488", "rc": 0, "start": "2021-11-11 20:42:04.686005", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:42:04.772 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:04 +0000 (0:00:00.033) 0:04:25.028 ***** 2021-11-11T20:42:04.979 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Build SELinux policy package] **************************** 2021-11-11T20:42:04.981 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": ["semodule_package", "-o", "/tmp/nrpe.pp", "-m", "/tmp/nrpe.mod"], "delta": "0:00:00.012556", "end": "2021-11-11 20:42:04.954260", "rc": 0, "start": "2021-11-11 20:42:04.941704", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:42:05.002 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:04 +0000 (0:00:00.229) 0:04:25.258 ***** 2021-11-11T20:42:05.231 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Copy SELinux type enforcement file] ********************** 2021-11-11T20:42:05.233 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "99d9b2ea41fb2d6613898129718064833d2de428", "dest": "/tmp/nrpe.te", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/tmp/nrpe.te", "secontext": "unconfined_u:object_r:user_home_t:s0", "size": 934, "state": "file", "uid": 0} 2021-11-11T20:42:05.265 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:05 +0000 (0:00:00.262) 0:04:25.520 ***** 2021-11-11T20:42:05.465 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Compile SELinux module file] ***************************** 2021-11-11T20:42:05.466 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": ["checkmodule", "-M", "-m", "-o", "/tmp/nrpe.mod", "/tmp/nrpe.te"], "delta": "0:00:00.008162", "end": "2021-11-11 20:42:05.442144", "rc": 0, "start": "2021-11-11 20:42:05.433982", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:42:05.487 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:05 +0000 (0:00:00.222) 0:04:25.743 ***** 2021-11-11T20:42:05.683 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Build SELinux policy package] **************************** 2021-11-11T20:42:05.685 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": ["semodule_package", "-o", "/tmp/nrpe.pp", "-m", "/tmp/nrpe.mod"], "delta": "0:00:00.011546", "end": "2021-11-11 20:42:05.665944", "rc": 0, "start": "2021-11-11 20:42:05.654398", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:42:05.706 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:05 +0000 (0:00:00.218) 0:04:25.962 ***** 2021-11-11T20:42:20.311 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Load SELinux policy package] ***************************** 2021-11-11T20:42:20.313 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": ["semodule", "-i", "/tmp/nrpe.pp"], "delta": "0:00:15.123786", "end": "2021-11-11 20:42:20.289234", "rc": 0, "start": "2021-11-11 20:42:05.165448", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:42:20.335 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:20 +0000 (0:00:14.629) 0:04:40.591 ***** 2021-11-11T20:42:20.530 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Remove temporary files] ********************************** 2021-11-11T20:42:20.540 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "path": "/tmp/nrpe.*", "state": "absent"} 2021-11-11T20:42:20.552 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:20 +0000 (0:00:00.217) 0:04:40.808 ***** 2021-11-11T20:42:20.581 INFO:teuthology.task.ansible.out: TASK [testnode : Include package type specific vars.] ************************** ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"ceph_dependency_packages_to_remove": ["boost-random", "boost-program-options", "leveldb", "xmlstarlet", "python-jinja2", "python-ceph", "python-flask", "python-requests", "boost-random", "python-urllib3", "python-babel", "hdparm", "python-markupsafe", "python-werkzeug", "python-itsdangerous"], "ceph_packages_to_remove": ["ceph", "ceph-base", "ceph-selinux", "ceph-common", "ceph-debuginfo", "ceph-release", "libcephfs1", "ceph-radosgw", "python-ceph", "python-rados", "python-rbd", "python-cephfs", "librbd1", "librados2", "mod_fastcgi"], "dash_stream": "", "ntp_service_name": "ntpd", "packages_to_remove": ["device-mapper-multipath"], "pip_packages_to_install": ["remoto>=0.0.35"], "ssh_service_name": "sshd"}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/dnf_systems.yml"], "changed": false} 2021-11-11T20:42:20.606 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:20 +0000 (0:00:00.050) 0:04:40.859 ***** 2021-11-11T20:42:20.641 INFO:teuthology.task.ansible.out: TASK [testnode : Including distro specific variables.] ************************* 2021-11-11T20:42:20.643 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml) => {"ansible_facts": {}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml"], "ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml"} 2021-11-11T20:42:20.668 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:20 +0000 (0:00:00.064) 0:04:40.924 ***** 2021-11-11T20:42:20.709 INFO:teuthology.task.ansible.out: TASK [testnode : Including major version specific variables.] ****************** 2021-11-11T20:42:20.711 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/redhat_8.yml) => {"ansible_facts": {"common_yum_repos": {"apache-arrow": {"baseurl": "https://apache.jfrog.io/artifactory/arrow/centos/8/$basearch", "enabled": 1, "gpgcheck": 1, "name": "Apache Arrow"}, "lab-extras": {"baseurl": "http://{{ mirror_host }}/lab-extras/8/", "enabled": 1, "gpgcheck": 0, "name": "lab-extras"}}, "copr_repos": ["ceph/python3-asyncssh"], "epel_packages": ["dbench"], "nfs_service": "nfs-server", "ntp_service_name": "chronyd", "packages": ["dnf-utils", "git-all", "sysstat", "libedit", "boost-thread", "xfsprogs", "gdisk", "parted", "libgcrypt", "fuse-libs", "openssl", "libuuid", "attr", "ant", "lsof", "gettext", "bc", "xfsdump", "blktrace", "usbredir", "podman", "redhat-lsb", "firewalld", "wget", "libev-devel", "valgrind", "nfs-utils", "ncurses-devel", "gcc", "git", "make", "python3-nose", "python3-virtualenv", "genisoimage", "qemu-img", "qemu-kvm-core", "qemu-kvm-block-rbd", "libacl-devel", "autoconf", "gdb", "iozone", "lvm2"]}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/redhat_8.yml"], "ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/redhat_8.yml"} 2021-11-11T20:42:20.735 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:20 +0000 (0:00:00.067) 0:04:40.991 ***** 2021-11-11T20:42:20.778 INFO:teuthology.task.ansible.out: TASK [testnode : Including version specific variables.] ************************ 2021-11-11T20:42:20.780 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml) => {"ansible_facts": {}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml"], "ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml"} 2021-11-11T20:42:20.801 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:20 +0000 (0:00:00.065) 0:04:41.057 ***** 2021-11-11T20:42:20.914 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Load SELinux policy package] ***************************** changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": ["semodule", "-i", "/tmp/nrpe.pp"], "delta": "0:00:15.009380", "end": "2021-11-11 20:42:20.893195", "rc": 0, "start": "2021-11-11 20:42:05.883815", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:42:20.937 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:20 +0000 (0:00:00.136) 0:04:41.193 ***** 2021-11-11T20:42:20.985 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure the sudo group exists.] ******************************** 2021-11-11T20:42:20.987 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "gid": 1000, "name": "sudo", "state": "present", "system": false} 2021-11-11T20:42:21.009 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.071) 0:04:41.265 ***** 2021-11-11T20:42:21.143 INFO:teuthology.task.ansible.out: TASK [common : nrpe - Remove temporary files] ********************************** 2021-11-11T20:42:21.145 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "path": "/tmp/nrpe.*", "state": "absent"} 2021-11-11T20:42:21.167 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.158) 0:04:41.423 ***** 2021-11-11T20:42:21.195 INFO:teuthology.task.ansible.out: TASK [testnode : Include package type specific vars.] ************************** ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"ceph_dependency_packages_to_remove": ["boost-random", "boost-program-options", "leveldb", "xmlstarlet", "python-jinja2", "python-ceph", "python-flask", "python-requests", "boost-random", "python-urllib3", "python-babel", "hdparm", "python-markupsafe", "python-werkzeug", "python-itsdangerous"], "ceph_packages_to_remove": ["ceph", "ceph-base", "ceph-selinux", "ceph-common", "ceph-debuginfo", "ceph-release", "libcephfs1", "ceph-radosgw", "python-ceph", "python-rados", "python-rbd", "python-cephfs", "librbd1", "librados2", "mod_fastcgi"], "dash_stream": "", "ntp_service_name": "ntpd", "packages_to_remove": ["device-mapper-multipath"], "pip_packages_to_install": ["remoto>=0.0.35"], "ssh_service_name": "sshd"}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/dnf_systems.yml"], "changed": false} 2021-11-11T20:42:21.198 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure the teuthology_user group exists.] ********************* ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "gid": 1238, "name": "ubuntu", "state": "present", "system": false} 2021-11-11T20:42:21.218 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.050) 0:04:41.474 ***** 2021-11-11T20:42:21.239 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.020) 0:04:41.495 ***** 2021-11-11T20:42:21.246 INFO:teuthology.task.ansible.out: TASK [testnode : Create the teuthology user.] ********************************** 2021-11-11T20:42:21.248 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:21.271 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.032) 0:04:41.527 ***** 2021-11-11T20:42:21.289 INFO:teuthology.task.ansible.out: TASK [testnode : Including distro specific variables.] ************************* 2021-11-11T20:42:21.291 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml) => {"ansible_facts": {}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml"], "ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml"} 2021-11-11T20:42:21.317 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.045) 0:04:41.573 ***** 2021-11-11T20:42:21.324 INFO:teuthology.task.ansible.out: TASK [testnode : Delete the teuthology users password.] ************************ 2021-11-11T20:42:21.326 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:21.349 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.031) 0:04:41.604 ***** 2021-11-11T20:42:21.358 INFO:teuthology.task.ansible.out: TASK [testnode : Including major version specific variables.] ****************** 2021-11-11T20:42:21.359 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/redhat_8.yml) => {"ansible_facts": {"common_yum_repos": {"apache-arrow": {"baseurl": "https://apache.jfrog.io/artifactory/arrow/centos/8/$basearch", "enabled": 1, "gpgcheck": 1, "name": "Apache Arrow"}, "lab-extras": {"baseurl": "http://{{ mirror_host }}/lab-extras/8/", "enabled": 1, "gpgcheck": 0, "name": "lab-extras"}}, "copr_repos": ["ceph/python3-asyncssh"], "epel_packages": ["dbench"], "nfs_service": "nfs-server", "ntp_service_name": "chronyd", "packages": ["dnf-utils", "git-all", "sysstat", "libedit", "boost-thread", "xfsprogs", "gdisk", "parted", "libgcrypt", "fuse-libs", "openssl", "libuuid", "attr", "ant", "lsof", "gettext", "bc", "xfsdump", "blktrace", "usbredir", "podman", "redhat-lsb", "firewalld", "wget", "libev-devel", "valgrind", "nfs-utils", "ncurses-devel", "gcc", "git", "make", "python3-nose", "python3-virtualenv", "genisoimage", "qemu-img", "qemu-kvm-core", "qemu-kvm-block-rbd", "libacl-devel", "autoconf", "gdb", "iozone", "lvm2"]}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/redhat_8.yml"], "ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/redhat_8.yml"} 2021-11-11T20:42:21.387 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.037) 0:04:41.642 ***** 2021-11-11T20:42:21.447 INFO:teuthology.task.ansible.out: TASK [testnode : Including version specific variables.] ************************ 2021-11-11T20:42:21.449 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml) => {"ansible_facts": {}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml"], "ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml"} 2021-11-11T20:42:21.475 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.088) 0:04:41.731 ***** 2021-11-11T20:42:21.566 INFO:teuthology.task.ansible.out: TASK [testnode : Add a user for xfstests to test user quotas.] ***************** 2021-11-11T20:42:21.568 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"append": false, "changed": false, "comment": "", "group": 10101, "home": "/home/fsgqa", "move_home": false, "name": "fsgqa", "shell": "/bin/bash", "state": "present", "uid": 10101} 2021-11-11T20:42:21.591 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.116) 0:04:41.847 ***** 2021-11-11T20:42:21.673 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure the sudo group exists.] ******************************** ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "gid": 1000, "name": "sudo", "state": "present", "system": false} 2021-11-11T20:42:21.696 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.104) 0:04:41.952 ***** 2021-11-11T20:42:21.891 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure the teuthology_user group exists.] ********************* 2021-11-11T20:42:21.893 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "gid": 1238, "name": "ubuntu", "state": "present", "system": false} 2021-11-11T20:42:21.913 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.217) 0:04:42.169 ***** 2021-11-11T20:42:21.941 INFO:teuthology.task.ansible.out: TASK [testnode : Create the teuthology user.] ********************************** 2021-11-11T20:42:21.942 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:21.965 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:21 +0000 (0:00:00.050) 0:04:42.220 ***** 2021-11-11T20:42:21.991 INFO:teuthology.task.ansible.out: TASK [testnode : Delete the teuthology users password.] ************************ 2021-11-11T20:42:21.993 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:22.014 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:22 +0000 (0:00:00.050) 0:04:42.270 ***** 2021-11-11T20:42:22.129 INFO:teuthology.task.ansible.out: TASK [testnode : Set a high max open files limit for the teuthology user.] ***** 2021-11-11T20:42:22.130 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "dc140f4bb26594b8d5e0753bab8d93c0cced7772", "dest": "/etc/security/limits.d/ubuntu.conf", "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/etc/security/limits.d/ubuntu.conf", "secontext": "system_u:object_r:etc_t:s0", "size": 112, "state": "file", "uid": 0} 2021-11-11T20:42:22.151 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:22 +0000 (0:00:00.136) 0:04:42.407 ***** 2021-11-11T20:42:22.190 INFO:teuthology.task.ansible.out: TASK [testnode : Set hostname var] ********************************************* 2021-11-11T20:42:22.192 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"hostname": "smithi096"}, "changed": false} 2021-11-11T20:42:22.212 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:22 +0000 (0:00:00.060) 0:04:42.468 ***** 2021-11-11T20:42:22.230 INFO:teuthology.task.ansible.out: TASK [testnode : Add a user for xfstests to test user quotas.] ***************** 2021-11-11T20:42:22.232 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"append": false, "changed": false, "comment": "", "group": 10101, "home": "/home/fsgqa", "move_home": false, "name": "fsgqa", "shell": "/bin/bash", "state": "present", "uid": 10101} 2021-11-11T20:42:22.255 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:22 +0000 (0:00:00.042) 0:04:42.511 ***** 2021-11-11T20:42:22.798 INFO:teuthology.task.ansible.out: TASK [testnode : Set a high max open files limit for the teuthology user.] ***** ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "dc140f4bb26594b8d5e0753bab8d93c0cced7772", "dest": "/etc/security/limits.d/ubuntu.conf", "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/etc/security/limits.d/ubuntu.conf", "secontext": "system_u:object_r:etc_t:s0", "size": 112, "state": "file", "uid": 0} 2021-11-11T20:42:22.820 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:22 +0000 (0:00:00.565) 0:04:43.076 ***** 2021-11-11T20:42:22.858 INFO:teuthology.task.ansible.out: TASK [testnode : Set hostname var] ********************************************* 2021-11-11T20:42:22.865 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"hostname": "smithi198"}, "changed": false} 2021-11-11T20:42:22.880 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:22 +0000 (0:00:00.060) 0:04:43.136 ***** 2021-11-11T20:42:22.972 INFO:teuthology.task.ansible.out: TASK [testnode : Set the system's hostname] ************************************ 2021-11-11T20:42:22.974 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"ansible_domain": "", "ansible_fqdn": "smithi096", "ansible_hostname": "smithi096", "ansible_nodename": "smithi096"}, "changed": false, "name": "smithi096"} 2021-11-11T20:42:22.996 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:22 +0000 (0:00:00.115) 0:04:43.252 ***** 2021-11-11T20:42:23.320 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"ansible_domain": "", "ansible_fqdn": "smithi198", "ansible_hostname": "smithi198", "ansible_nodename": "smithi198"}, "changed": false, "name": "smithi198"} 2021-11-11T20:42:23.348 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:23 +0000 (0:00:00.351) 0:04:43.604 ***** 2021-11-11T20:42:23.451 INFO:teuthology.task.ansible.out: TASK [testnode : Upload distro major version specific sshd_config] ************* ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "a5e550b2aa74f6355e926d70d73a50ad9ab4b4be", "dest": "/etc/ssh/sshd_config", "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/etc/ssh/sshd_config", "secontext": "system_u:object_r:etc_t:s0", "size": 1043, "state": "file", "uid": 0} 2021-11-11T20:42:23.475 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:23 +0000 (0:00:00.126) 0:04:43.731 ***** 2021-11-11T20:42:23.795 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "a5e550b2aa74f6355e926d70d73a50ad9ab4b4be", "dest": "/etc/ssh/sshd_config", "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/etc/ssh/sshd_config", "secontext": "system_u:object_r:etc_t:s0", "size": 1043, "state": "file", "uid": 0} 2021-11-11T20:42:23.818 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:23 +0000 (0:00:00.343) 0:04:44.074 ***** 2021-11-11T20:42:23.929 INFO:teuthology.task.ansible.out: TASK [testnode : Upload ssh_config] ******************************************** 2021-11-11T20:42:23.931 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "58d78b02605c8368f1913de12e145f4ad930fafc", "dest": "/etc/ssh/ssh_config", "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/etc/ssh/ssh_config", "secontext": "system_u:object_r:etc_t:s0", "size": 570, "state": "file", "uid": 0} 2021-11-11T20:42:23.953 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:23 +0000 (0:00:00.135) 0:04:44.209 ***** 2021-11-11T20:42:24.275 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "58d78b02605c8368f1913de12e145f4ad930fafc", "dest": "/etc/ssh/ssh_config", "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/etc/ssh/ssh_config", "secontext": "system_u:object_r:etc_t:s0", "size": 570, "state": "file", "uid": 0} 2021-11-11T20:42:24.300 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:24 +0000 (0:00:00.345) 0:04:44.555 ***** 2021-11-11T20:42:24.568 INFO:teuthology.task.ansible.out: TASK [testnode : Add ssh pubkeys] ********************************************** 2021-11-11T20:42:24.570 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"attempts": 1, "changed": true, "comment": null, "exclusive": false, "follow": false, "key": "https://raw.githubusercontent.com/ceph/keys/autogenerated/ssh/@all.pub", "key_options": null, "keyfile": "/home/ubuntu/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ubuntu", "validate_certs": true} 2021-11-11T20:42:24.591 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:24 +0000 (0:00:00.291) 0:04:44.847 ***** 2021-11-11T20:42:24.859 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"attempts": 1, "changed": true, "comment": null, "exclusive": false, "follow": false, "key": "https://raw.githubusercontent.com/ceph/keys/autogenerated/ssh/@all.pub", "key_options": null, "keyfile": "/home/ubuntu/.ssh/authorized_keys", "manage_dir": true, "path": null, "state": "present", "user": "ubuntu", "validate_certs": true} 2021-11-11T20:42:24.885 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:24 +0000 (0:00:00.293) 0:04:45.141 ***** 2021-11-11T20:42:25.062 INFO:teuthology.task.ansible.out: TASK [testnode : Create remote.conf] ******************************************* 2021-11-11T20:42:25.064 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "68d0b8ee2f88a77895f257aa69f32a42e47f455b", "dest": "/etc/security/limits.d/remote.conf", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/security/limits.d/remote.conf", "secontext": "system_u:object_r:etc_t:s0", "size": 116, "state": "file", "uid": 0} 2021-11-11T20:42:25.086 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:25 +0000 (0:00:00.200) 0:04:45.341 ***** 2021-11-11T20:42:25.284 INFO:teuthology.task.ansible.out: TASK [testnode : Set mode on /etc/fuse.conf] *********************************** 2021-11-11T20:42:25.286 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "dest": "/etc/fuse.conf", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "secontext": "unconfined_u:object_r:etc_t:s0", "size": 0, "state": "file", "uid": 0} 2021-11-11T20:42:25.306 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:25 +0000 (0:00:00.220) 0:04:45.562 ***** 2021-11-11T20:42:25.354 INFO:teuthology.task.ansible.out: TASK [testnode : Create remote.conf] ******************************************* 2021-11-11T20:42:25.356 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "68d0b8ee2f88a77895f257aa69f32a42e47f455b", "dest": "/etc/security/limits.d/remote.conf", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/security/limits.d/remote.conf", "secontext": "system_u:object_r:etc_t:s0", "size": 116, "state": "file", "uid": 0} 2021-11-11T20:42:25.378 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:25 +0000 (0:00:00.072) 0:04:45.634 ***** 2021-11-11T20:42:25.500 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure the group kvm exists.] ********************************* 2021-11-11T20:42:25.502 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "gid": 36, "name": "kvm", "state": "present", "system": false} 2021-11-11T20:42:25.528 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:25 +0000 (0:00:00.149) 0:04:45.784 ***** 2021-11-11T20:42:25.588 INFO:teuthology.task.ansible.out: TASK [testnode : Set mode on /etc/fuse.conf] *********************************** 2021-11-11T20:42:25.590 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "dest": "/etc/fuse.conf", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "secontext": "unconfined_u:object_r:etc_t:s0", "size": 0, "state": "file", "uid": 0} 2021-11-11T20:42:25.612 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:25 +0000 (0:00:00.084) 0:04:45.868 ***** 2021-11-11T20:42:25.745 INFO:teuthology.task.ansible.out: TASK [testnode : Add the teuthology user to groups kvm,disk] ******************* 2021-11-11T20:42:25.747 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"append": true, "changed": false, "comment": "", "group": 1238, "groups": "kvm,disk", "home": "/home/ubuntu", "move_home": false, "name": "ubuntu", "shell": "/bin/bash", "state": "present", "uid": 1000} 2021-11-11T20:42:25.769 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:25 +0000 (0:00:00.156) 0:04:46.025 ***** 2021-11-11T20:42:25.807 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure the group kvm exists.] ********************************* 2021-11-11T20:42:25.809 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "gid": 36, "name": "kvm", "state": "present", "system": false} 2021-11-11T20:42:25.833 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:25 +0000 (0:00:00.063) 0:04:46.089 ***** 2021-11-11T20:42:26.059 INFO:teuthology.task.ansible.out: TASK [testnode : Add the teuthology user to groups kvm,disk] ******************* 2021-11-11T20:42:26.061 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"append": true, "changed": false, "comment": "", "group": 1238, "groups": "kvm,disk", "home": "/home/ubuntu", "move_home": false, "name": "ubuntu", "shell": "/bin/bash", "state": "present", "uid": 1000} 2021-11-11T20:42:26.082 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:26 +0000 (0:00:00.248) 0:04:46.338 ***** 2021-11-11T20:42:26.235 INFO:teuthology.task.ansible.out: TASK [testnode : Configure /etc/sudoers.] ************************************** 2021-11-11T20:42:26.237 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "be3223c8b0917bde02657a5f9ddd95277cc3f812", "dest": "/etc/sudoers", "gid": 0, "group": "root", "mode": "0440", "owner": "root", "path": "/etc/sudoers", "secontext": "system_u:object_r:etc_t:s0", "size": 1808, "state": "file", "uid": 0} 2021-11-11T20:42:26.260 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:26 +0000 (0:00:00.177) 0:04:46.515 ***** 2021-11-11T20:42:26.551 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "be3223c8b0917bde02657a5f9ddd95277cc3f812", "dest": "/etc/sudoers", "gid": 0, "group": "root", "mode": "0440", "owner": "root", "path": "/etc/sudoers", "secontext": "system_u:object_r:etc_t:s0", "size": 1808, "state": "file", "uid": 0} 2021-11-11T20:42:26.575 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:26 +0000 (0:00:00.316) 0:04:46.831 ***** 2021-11-11T20:42:26.719 INFO:teuthology.task.ansible.out: TASK [testnode : Configure /etc/security/limits.conf] ************************** ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "12a2e17405b9e00ea2061e25929905d08e5fe19f", "dest": "/etc/security/limits.conf", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/security/limits.conf", "secontext": "system_u:object_r:etc_t:s0", "size": 854, "state": "file", "uid": 0} 2021-11-11T20:42:26.745 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:26 +0000 (0:00:00.169) 0:04:47.001 ***** 2021-11-11T20:42:27.048 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "12a2e17405b9e00ea2061e25929905d08e5fe19f", "dest": "/etc/security/limits.conf", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/security/limits.conf", "secontext": "system_u:object_r:etc_t:s0", "size": 854, "state": "file", "uid": 0} 2021-11-11T20:42:27.072 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:27 +0000 (0:00:00.326) 0:04:47.328 ***** 2021-11-11T20:42:29.043 INFO:teuthology.task.ansible.out: TASK [testnode : Rebuild rpmdb] ************************************************ [WARNING]: Consider using the yum, dnf or zypper module rather than running 'rpm'. If you need to use command because yum, dnf or zypper is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message. 2021-11-11T20:42:29.045 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": ["rpm", "--rebuilddb"], "delta": "0:00:02.111559", "end": "2021-11-11 20:42:29.021313", "rc": 0, "start": "2021-11-11 20:42:26.909754", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:42:29.065 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:29 +0000 (0:00:01.993) 0:04:49.321 ***** 2021-11-11T20:42:29.093 INFO:teuthology.task.ansible.out: TASK [testnode : Clean up Stream distro-provided repos] ************************ 2021-11-11T20:42:29.095 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:29.115 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:29 +0000 (0:00:00.049) 0:04:49.371 ***** 2021-11-11T20:42:29.153 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** 2021-11-11T20:42:29.155 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:29.176 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:29 +0000 (0:00:00.060) 0:04:49.432 ***** 2021-11-11T20:42:29.213 INFO:teuthology.task.ansible.out: TASK [testnode : Include CentOS Stream specific variables] ********************* 2021-11-11T20:42:29.215 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml) => {"ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml", "skip_reason": "Conditional result was False"} 2021-11-11T20:42:29.236 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:29 +0000 (0:00:00.060) 0:04:49.492 ***** 2021-11-11T20:42:29.263 INFO:teuthology.task.ansible.out: TASK [testnode : Configure local mirrorlists] ********************************** 2021-11-11T20:42:29.265 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:29.285 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:29 +0000 (0:00:00.049) 0:04:49.541 ***** 2021-11-11T20:42:29.472 INFO:teuthology.task.ansible.out: TASK [testnode : Rebuild rpmdb] ************************************************ 2021-11-11T20:42:29.474 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": ["rpm", "--rebuilddb"], "delta": "0:00:02.212347", "end": "2021-11-11 20:42:29.450138", "rc": 0, "start": "2021-11-11 20:42:27.237791", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:42:29.496 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:29 +0000 (0:00:00.210) 0:04:49.752 ***** 2021-11-11T20:42:29.524 INFO:teuthology.task.ansible.out: TASK [testnode : Clean up Stream distro-provided repos] ************************ 2021-11-11T20:42:29.526 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:29.546 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:29 +0000 (0:00:00.050) 0:04:49.802 ***** 2021-11-11T20:42:29.584 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** 2021-11-11T20:42:29.586 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:29.606 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:29 +0000 (0:00:00.059) 0:04:49.862 ***** 2021-11-11T20:42:29.643 INFO:teuthology.task.ansible.out: TASK [testnode : Include CentOS Stream specific variables] ********************* 2021-11-11T20:42:29.645 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml) => {"ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/testnode/vars/empty.yml", "skip_reason": "Conditional result was False"} 2021-11-11T20:42:29.667 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:29 +0000 (0:00:00.061) 0:04:49.923 ***** 2021-11-11T20:42:29.705 INFO:teuthology.task.ansible.out: TASK [testnode : Configure local mirrorlists] ********************************** 2021-11-11T20:42:29.706 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:29.727 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:29 +0000 (0:00:00.059) 0:04:49.983 ***** 2021-11-11T20:42:29.765 INFO:teuthology.task.ansible.out: TASK [testnode : Configure common additional repos in /etc/yum.repos.d/] ******* 2021-11-11T20:42:29.767 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'key': 'lab-extras', 'value': {'name': 'lab-extras', 'baseurl': 'http://apt-mirror.front.sepia.ceph.com/lab-extras/8/', 'enabled': 1, 'gpgcheck': 0}}) => {"ansible_loop_var": "item", "changed": false, "checksum": "a0e2ac0628a264e14b96b1c57627a3aa344a6c52", "dest": "/etc/yum.repos.d/lab-extras.repo", "gid": 0, "group": "root", "item": {"key": "lab-extras", "value": {"baseurl": "http://apt-mirror.front.sepia.ceph.com/lab-extras/8/", "enabled": 1, "gpgcheck": 0, "name": "lab-extras"}}, "mode": "0644", "owner": "root", "path": "/etc/yum.repos.d/lab-extras.repo", "secontext": "system_u:object_r:system_conf_t:s0", "size": 203, "state": "file", "uid": 0} 2021-11-11T20:42:30.224 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'key': 'lab-extras', 'value': {'name': 'lab-extras', 'baseurl': 'http://apt-mirror.front.sepia.ceph.com/lab-extras/8/', 'enabled': 1, 'gpgcheck': 0}}) => {"ansible_loop_var": "item", "changed": false, "checksum": "a0e2ac0628a264e14b96b1c57627a3aa344a6c52", "dest": "/etc/yum.repos.d/lab-extras.repo", "gid": 0, "group": "root", "item": {"key": "lab-extras", "value": {"baseurl": "http://apt-mirror.front.sepia.ceph.com/lab-extras/8/", "enabled": 1, "gpgcheck": 0, "name": "lab-extras"}}, "mode": "0644", "owner": "root", "path": "/etc/yum.repos.d/lab-extras.repo", "secontext": "system_u:object_r:system_conf_t:s0", "size": 203, "state": "file", "uid": 0} 2021-11-11T20:42:30.257 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'apache-arrow', 'value': {'name': 'Apache Arrow', 'baseurl': 'https://apache.jfrog.io/artifactory/arrow/centos/8/$basearch', 'enabled': 1, 'gpgcheck': 1}}) => {"ansible_loop_var": "item", "changed": true, "checksum": "86b924db6e383ddad921a207c09a708e6cca6a67", "dest": "/etc/yum.repos.d/apache-arrow.repo", "gid": 0, "group": "root", "item": {"key": "apache-arrow", "value": {"baseurl": "https://apache.jfrog.io/artifactory/arrow/centos/8/$basearch", "enabled": 1, "gpgcheck": 1, "name": "Apache Arrow"}}, "md5sum": "183d0889f7213a1e38c7f48965d7549e", "mode": "0644", "owner": "root", "secontext": "system_u:object_r:system_conf_t:s0", "size": 215, "src": "/home/ubuntu/.ansible/tmp/ansible-tmp-1636663349.8004436-17176-23819051765832/source", "state": "file", "uid": 0} 2021-11-11T20:42:30.280 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:30 +0000 (0:00:00.553) 0:04:50.536 ***** 2021-11-11T20:42:30.308 INFO:teuthology.task.ansible.out: TASK [testnode : Configure version specific repos in /etc/yum.repos.d/] ******** 2021-11-11T20:42:30.331 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:30 +0000 (0:00:00.050) 0:04:50.586 ***** 2021-11-11T20:42:30.703 INFO:teuthology.task.ansible.out: TASK [testnode : Configure common additional repos in /etc/yum.repos.d/] ******* 2021-11-11T20:42:30.705 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'apache-arrow', 'value': {'name': 'Apache Arrow', 'baseurl': 'https://apache.jfrog.io/artifactory/arrow/centos/8/$basearch', 'enabled': 1, 'gpgcheck': 1}}) => {"ansible_loop_var": "item", "changed": true, "checksum": "86b924db6e383ddad921a207c09a708e6cca6a67", "dest": "/etc/yum.repos.d/apache-arrow.repo", "gid": 0, "group": "root", "item": {"key": "apache-arrow", "value": {"baseurl": "https://apache.jfrog.io/artifactory/arrow/centos/8/$basearch", "enabled": 1, "gpgcheck": 1, "name": "Apache Arrow"}}, "md5sum": "183d0889f7213a1e38c7f48965d7549e", "mode": "0644", "owner": "root", "secontext": "system_u:object_r:system_conf_t:s0", "size": 215, "src": "/home/ubuntu/.ansible/tmp/ansible-tmp-1636663350.251581-17234-143478493400697/source", "state": "file", "uid": 0} 2021-11-11T20:42:30.729 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:30 +0000 (0:00:00.398) 0:04:50.985 ***** 2021-11-11T20:42:30.756 INFO:teuthology.task.ansible.out: TASK [testnode : Configure version specific repos in /etc/yum.repos.d/] ******** 2021-11-11T20:42:30.778 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:30 +0000 (0:00:00.049) 0:04:51.034 ***** 2021-11-11T20:42:31.227 INFO:teuthology.task.ansible.out: TASK [testnode : Enable copr repos] ******************************************** 2021-11-11T20:42:31.229 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item=ceph/python3-asyncssh) => {"ansible_loop_var": "item", "changed": true, "cmd": ["dnf", "-y", "copr", "enable", "ceph/python3-asyncssh"], "delta": "0:00:00.695304", "end": "2021-11-11 20:42:31.203476", "item": "ceph/python3-asyncssh", "rc": 0, "start": "2021-11-11 20:42:30.508172", "stderr": "Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.", "stderr_lines": ["Enabling a Copr repository. Please note that this repository is not part", "of the main distribution, and quality may vary.", "", "The Fedora Project does not exercise any power over the contents of", "this repository beyond the rules outlined in the Copr FAQ at", ",", "and packages are not held to any quality or security level.", "", "Please do not file bug reports about these packages in Fedora", "Bugzilla. In case of problems, contact the owner of this repository."], "stdout": "Updating Subscription Management repositories.\nRepository successfully enabled.", "stdout_lines": ["Updating Subscription Management repositories.", "Repository successfully enabled."], "warnings": ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]} [WARNING]: Consider using the dnf module rather than running 'dnf'. If you need to use 2021-11-11T20:42:31.230 INFO:teuthology.task.ansible.out: command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message. 2021-11-11T20:42:31.251 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:31 +0000 (0:00:00.473) 0:04:51.507 ***** 2021-11-11T20:42:31.629 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item=ceph/python3-asyncssh) => {"ansible_loop_var": "item", "changed": true, "cmd": ["dnf", "-y", "copr", "enable", "ceph/python3-asyncssh"], "delta": "0:00:00.646634", "end": "2021-11-11 20:42:31.606516", "item": "ceph/python3-asyncssh", "rc": 0, "start": "2021-11-11 20:42:30.959882", "stderr": "Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.", "stderr_lines": ["Enabling a Copr repository. Please note that this repository is not part", "of the main distribution, and quality may vary.", "", "The Fedora Project does not exercise any power over the contents of", "this repository beyond the rules outlined in the Copr FAQ at", ",", "and packages are not held to any quality or security level.", "", "Please do not file bug reports about these packages in Fedora", "Bugzilla. In case of problems, contact the owner of this repository."], "stdout": "Updating Subscription Management repositories.\nRepository successfully enabled.", "stdout_lines": ["Updating Subscription Management repositories.", "Repository successfully enabled."], "warnings": ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]} 2021-11-11T20:42:31.652 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:31 +0000 (0:00:00.400) 0:04:51.908 ***** 2021-11-11T20:42:46.777 INFO:teuthology.task.ansible.out: TASK [testnode : Install GPG] ************************************************** 2021-11-11T20:42:46.779 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:42:46.800 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:46 +0000 (0:00:15.147) 0:05:07.056 ***** 2021-11-11T20:42:47.006 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:42:47.033 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:47 +0000 (0:00:00.233) 0:05:07.289 ***** 2021-11-11T20:42:47.619 INFO:teuthology.task.ansible.out: TASK [testnode : Install GPG keys] ********************************************* 2021-11-11T20:42:47.621 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item=https://download.ceph.com/keys/release.asc) => {"ansible_loop_var": "item", "changed": true, "item": "https://download.ceph.com/keys/release.asc"} 2021-11-11T20:42:47.673 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item=https://download.ceph.com/keys/release.asc) => {"ansible_loop_var": "item", "changed": true, "item": "https://download.ceph.com/keys/release.asc"} 2021-11-11T20:42:48.130 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item=https://download.ceph.com/keys/autobuild.asc) => {"ansible_loop_var": "item", "changed": true, "item": "https://download.ceph.com/keys/autobuild.asc"} 2021-11-11T20:42:48.183 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item=https://download.ceph.com/keys/autobuild.asc) => {"ansible_loop_var": "item", "changed": true, "item": "https://download.ceph.com/keys/autobuild.asc"} 2021-11-11T20:42:49.422 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item=https://dist.apache.org/repos/dist/dev/arrow/KEYS) => {"ansible_loop_var": "item", "changed": true, "item": "https://dist.apache.org/repos/dist/dev/arrow/KEYS"} 2021-11-11T20:42:49.446 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:49 +0000 (0:00:02.412) 0:05:09.702 ***** 2021-11-11T20:42:49.688 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item=https://dist.apache.org/repos/dist/dev/arrow/KEYS) => {"ansible_loop_var": "item", "changed": true, "item": "https://dist.apache.org/repos/dist/dev/arrow/KEYS"} 2021-11-11T20:42:49.712 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:49 +0000 (0:00:00.265) 0:05:09.968 ***** 2021-11-11T20:42:50.281 INFO:teuthology.task.ansible.out: TASK [testnode : Clean yum cache] ********************************************** 2021-11-11T20:42:50.282 INFO:teuthology.task.ansible.out:[WARNING]: Consider using the yum module rather than running 'yum'. If you need to use command because yum is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message. changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": "yum clean all", "delta": "0:00:00.637584", "end": "2021-11-11 20:42:50.264951", "rc": 0, "start": "2021-11-11 20:42:49.627367", "stderr": "", "stderr_lines": [], "stdout": "Updating Subscription Management repositories.\n56 files removed", "stdout_lines": ["Updating Subscription Management repositories.", "56 files removed"]} 2021-11-11T20:42:50.304 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:50 +0000 (0:00:00.591) 0:05:10.560 ***** 2021-11-11T20:42:50.332 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:50.354 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:50 +0000 (0:00:00.050) 0:05:10.610 ***** 2021-11-11T20:42:50.392 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure yum_utils is present.] ********************************* skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:50.414 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:50 +0000 (0:00:00.059) 0:05:10.670 ***** 2021-11-11T20:42:50.451 INFO:teuthology.task.ansible.out: TASK [testnode : Removing saved yum transactions] ****************************** 2021-11-11T20:42:50.453 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:50.474 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:50 +0000 (0:00:00.059) 0:05:10.730 ***** 2021-11-11T20:42:50.523 INFO:teuthology.task.ansible.out: TASK [testnode : Clean yum cache] ********************************************** 2021-11-11T20:42:50.525 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": "yum clean all", "delta": "0:00:00.607019", "end": "2021-11-11 20:42:50.503965", "rc": 0, "start": "2021-11-11 20:42:49.896946", "stderr": "", "stderr_lines": [], "stdout": "Updating Subscription Management repositories.\n56 files removed", "stdout_lines": ["Updating Subscription Management repositories.", "56 files removed"]} 2021-11-11T20:42:50.548 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:50 +0000 (0:00:00.073) 0:05:10.803 ***** 2021-11-11T20:42:50.576 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** 2021-11-11T20:42:50.577 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:50.598 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:50 +0000 (0:00:00.050) 0:05:10.854 ***** 2021-11-11T20:42:50.635 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure yum_utils is present.] ********************************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:50.658 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:50 +0000 (0:00:00.059) 0:05:10.914 ***** 2021-11-11T20:42:50.676 INFO:teuthology.task.ansible.out: TASK [testnode : Check if ceph-debuginfo is installed] ************************* 2021-11-11T20:42:50.678 INFO:teuthology.task.ansible.out:fatal: [smithi096.front.sepia.ceph.com]: FAILED! => {"changed": false, "cmd": ["rpm", "-q", "ceph-debuginfo"], "delta": "0:00:00.007639", "end": "2021-11-11 20:42:50.649535", "msg": "non-zero return code", "rc": 1, "start": "2021-11-11 20:42:50.641896", "stderr": "", "stderr_lines": [], "stdout": "package ceph-debuginfo is not installed", "stdout_lines": ["package ceph-debuginfo is not installed"]} ...ignoring 2021-11-11T20:42:50.700 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:50 +0000 (0:00:00.041) 0:05:10.956 ***** 2021-11-11T20:42:50.708 INFO:teuthology.task.ansible.out: TASK [testnode : Removing saved yum transactions] ****************************** 2021-11-11T20:42:50.709 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:50.732 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:50 +0000 (0:00:00.032) 0:05:10.988 ***** 2021-11-11T20:42:50.739 INFO:teuthology.task.ansible.out: TASK [testnode : Work around https://bugzilla.redhat.com/show_bug.cgi?id=1234967] *** 2021-11-11T20:42:50.741 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:50.764 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:50 +0000 (0:00:00.031) 0:05:11.020 ***** 2021-11-11T20:42:50.927 INFO:teuthology.task.ansible.out: TASK [testnode : Check if ceph-debuginfo is installed] ************************* 2021-11-11T20:42:50.929 INFO:teuthology.task.ansible.out:fatal: [smithi198.front.sepia.ceph.com]: FAILED! => {"changed": false, "cmd": ["rpm", "-q", "ceph-debuginfo"], "delta": "0:00:00.007006", "end": "2021-11-11 20:42:50.908424", "msg": "non-zero return code", "rc": 1, "start": "2021-11-11 20:42:50.901418", "stderr": "", "stderr_lines": [], "stdout": "package ceph-debuginfo is not installed", "stdout_lines": ["package ceph-debuginfo is not installed"]} ...ignoring 2021-11-11T20:42:50.952 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:50 +0000 (0:00:00.188) 0:05:11.208 ***** 2021-11-11T20:42:50.991 INFO:teuthology.task.ansible.out: TASK [testnode : Work around https://bugzilla.redhat.com/show_bug.cgi?id=1234967] *** 2021-11-11T20:42:50.993 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:42:51.018 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:42:51 +0000 (0:00:00.065) 0:05:11.274 ***** 2021-11-11T20:43:25.985 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure ceph packages are not present.] ************************ 2021-11-11T20:43:25.987 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "msg": "", "rc": 0, "results": ["Removed: librados2-1:12.2.7-9.el8.x86_64", "Removed: librbd1-1:12.2.7-9.el8.x86_64", "Removed: qemu-kvm-block-rbd-15:4.2.0-48.module+el8.4.0+10368+630e803b.x86_64"]} 2021-11-11T20:43:26.010 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:43:26 +0000 (0:00:34.992) 0:05:46.266 ***** 2021-11-11T20:43:26.537 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "msg": "", "rc": 0, "results": ["Removed: librados2-1:12.2.7-9.el8.x86_64", "Removed: librbd1-1:12.2.7-9.el8.x86_64", "Removed: qemu-kvm-block-rbd-15:4.2.0-48.module+el8.4.0+10368+630e803b.x86_64"]} 2021-11-11T20:43:26.561 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:43:26 +0000 (0:00:00.550) 0:05:46.817 ***** 2021-11-11T20:43:55.626 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure ceph dependency packages are not present.] ************* 2021-11-11T20:43:55.628 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "msg": "", "rc": 0, "results": ["Removed: boost-program-options-1.66.0-10.el8.x86_64", "Removed: boost-random-1.66.0-10.el8.x86_64"]} 2021-11-11T20:43:55.653 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:43:55 +0000 (0:00:29.090) 0:06:15.907 ***** 2021-11-11T20:43:56.184 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "msg": "", "rc": 0, "results": ["Removed: boost-program-options-1.66.0-10.el8.x86_64", "Removed: boost-random-1.66.0-10.el8.x86_64"]} 2021-11-11T20:43:56.208 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:43:56 +0000 (0:00:00.556) 0:06:16.464 ***** 2021-11-11T20:44:29.505 INFO:teuthology.task.ansible.out: TASK [testnode : Install packages] ********************************************* changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "msg": "", "rc": 0, "results": ["Installed: qemu-img-15:4.2.0-48.module+el8.4.0+11909+3300d70f.3.x86_64", "Installed: boost-random-1.66.0-10.el8.x86_64", "Installed: boost-program-options-1.66.0-10.el8.x86_64", "Installed: librados2-1:12.2.7-9.el8.x86_64", "Installed: librbd1-1:12.2.7-9.el8.x86_64", "Installed: qemu-kvm-common-15:4.2.0-48.module+el8.4.0+11909+3300d70f.3.x86_64", "Installed: qemu-kvm-core-15:4.2.0-48.module+el8.4.0+11909+3300d70f.3.x86_64", "Installed: qemu-kvm-block-rbd-15:4.2.0-48.module+el8.4.0+11909+3300d70f.3.x86_64", "Removed: qemu-img-15:4.2.0-48.module+el8.4.0+10368+630e803b.x86_64", "Removed: qemu-kvm-common-15:4.2.0-48.module+el8.4.0+10368+630e803b.x86_64", "Removed: qemu-kvm-core-15:4.2.0-48.module+el8.4.0+10368+630e803b.x86_64"]} 2021-11-11T20:44:29.530 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:29 +0000 (0:00:33.321) 0:06:49.786 ***** 2021-11-11T20:44:30.195 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "msg": "", "rc": 0, "results": ["Installed: qemu-img-15:4.2.0-48.module+el8.4.0+11909+3300d70f.3.x86_64", "Installed: boost-random-1.66.0-10.el8.x86_64", "Installed: boost-program-options-1.66.0-10.el8.x86_64", "Installed: librados2-1:12.2.7-9.el8.x86_64", "Installed: librbd1-1:12.2.7-9.el8.x86_64", "Installed: qemu-kvm-common-15:4.2.0-48.module+el8.4.0+11909+3300d70f.3.x86_64", "Installed: qemu-kvm-core-15:4.2.0-48.module+el8.4.0+11909+3300d70f.3.x86_64", "Installed: qemu-kvm-block-rbd-15:4.2.0-48.module+el8.4.0+11909+3300d70f.3.x86_64", "Removed: qemu-img-15:4.2.0-48.module+el8.4.0+10368+630e803b.x86_64", "Removed: qemu-kvm-common-15:4.2.0-48.module+el8.4.0+10368+630e803b.x86_64", "Removed: qemu-kvm-core-15:4.2.0-48.module+el8.4.0+10368+630e803b.x86_64"]} 2021-11-11T20:44:30.219 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:30 +0000 (0:00:00.688) 0:06:50.475 ***** 2021-11-11T20:44:43.652 INFO:teuthology.task.ansible.out: TASK [testnode : Install epel packages] **************************************** 2021-11-11T20:44:43.654 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:44:43.677 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:43 +0000 (0:00:13.458) 0:07:03.933 ***** 2021-11-11T20:44:44.273 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:44:44.301 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:44 +0000 (0:00:00.623) 0:07:04.557 ***** 2021-11-11T20:44:46.731 INFO:teuthology.task.ansible.out: TASK [testnode : Remove packages] ********************************************** 2021-11-11T20:44:46.733 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:44:46.755 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:46 +0000 (0:00:02.454) 0:07:07.011 ***** 2021-11-11T20:44:46.784 INFO:teuthology.task.ansible.out: TASK [testnode : Upgrade packages] ********************************************* skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:46.806 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:46 +0000 (0:00:00.050) 0:07:07.062 ***** 2021-11-11T20:44:47.117 INFO:teuthology.task.ansible.out: TASK [testnode : Stop and disable firewalld] *********************************** 2021-11-11T20:44:47.118 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "enabled": false, "name": "firewalld", "state": "stopped", "status": {"ActiveEnterTimestampMonotonic": "0", "ActiveExitTimestampMonotonic": "0", "ActiveState": "inactive", "After": "dbus.socket system.slice polkit.service dbus.service sysinit.target basic.target", "AllowIsolate": "no", "AllowedCPUs": "", "AllowedMemoryNodes": "", "AmbientCapabilities": "", "AssertResult": "no", "AssertTimestampMonotonic": "0", "Before": "shutdown.target network-pre.target", "BlockIOAccounting": "no", "BlockIOWeight": "[not set]", "BusName": "org.fedoraproject.FirewallD1", "CPUAccounting": "no", "CPUAffinity": "", "CPUAffinityFromNUMA": "no", "CPUQuotaPerSecUSec": "infinity", "CPUQuotaPeriodUSec": "infinity", "CPUSchedulingPolicy": "0", "CPUSchedulingPriority": "0", "CPUSchedulingResetOnFork": "no", "CPUShares": "[not set]", "CPUUsageNSec": "[not set]", "CPUWeight": "[not set]", "CacheDirectoryMode": "0755", "CanFreeze": "yes", "CanIsolate": "no", "CanReload": "yes", "CanStart": "yes", "CanStop": "yes", "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon", "CollectMode": "inactive", "ConditionResult": "no", "ConditionTimestampMonotonic": "0", "ConfigurationDirectoryMode": "0755", "Conflicts": "ebtables.service nftables.service ip6tables.service iptables.service ipset.service shutdown.target", "ControlPID": "0", "DefaultDependencies": "yes", "DefaultMemoryLow": "0", "DefaultMemoryMin": "0", "Delegate": "no", 2021-11-11T20:44:47.120 INFO:teuthology.task.ansible.out:"Description": "firewalld - dynamic firewall daemon", "DevicePolicy": "auto", "Documentation": "man:firewalld(1)", "DynamicUser": "no", "EffectiveCPUs": "", "EffectiveMemoryNodes": "", "EnvironmentFiles": "/etc/sysconfig/firewalld (ignore_errors=yes)", "ExecMainCode": "0", "ExecMainExitTimestampMonotonic": "0", "ExecMainPID": "0", "ExecMainStartTimestampMonotonic": "0", "ExecMainStatus": "0", "ExecReload": "{ path=/bin/kill ; argv[]=/bin/kill -HUP $MAINPID ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "ExecStart": "{ path=/usr/sbin/firewalld ; argv[]=/usr/sbin/firewalld --nofork --nopid $FIREWALLD_ARGS ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "FailureAction": "none", "FileDescriptorStoreMax": "0", "FragmentPath": "/usr/lib/systemd/system/firewalld.service", "FreezerState": "running", "GID": "[not set]", "GuessMainPID": "yes", "IOAccounting": "no", "IOSchedulingClass": "0", "IOSchedulingPriority": "0", "IOWeight": "[not set]", "IPAccounting": "no", "IPEgressBytes": "18446744073709551615", "IPEgressPackets": "18446744073709551615", "IPIngressBytes": "18446744073709551615", "IPIngressPackets": "18446744073709551615", "Id": "firewalld.service", "IgnoreOnIsolate": "no", "IgnoreSIGPIPE": "yes", "InactiveEnterTimestampMonotonic": "0", "InactiveExitTimestampMonotonic": "0", "JobRunningTimeoutUSec": "infinity", "JobTimeoutAction": "none", "JobTimeoutUSec": "infinity", "KeyringMode": "private", "KillMode": "mixed", "KillSignal": "15", "LimitAS": "infinity", "LimitASSoft": "infinity", "LimitCORE": "infinity", "LimitCORESoft": "infinity", "LimitCPU": "infinity", "LimitCPUSoft": "infinity", "LimitDATA": "infinity", "LimitDATASoft": "infinity", "LimitFSIZE": "infinity", "LimitFSIZESoft": "infinity", "LimitLOCKS": "infinity", "LimitLOCKSSoft": "infinity", "LimitMEMLOCK": "65536", "LimitMEMLOCKSoft": "65536", "LimitMSGQUEUE": "819200", "LimitMSGQUEUESoft": "819200", "LimitNICE": " 2021-11-11T20:44:47.122 INFO:teuthology.task.ansible.out:0", "LimitNICESoft": "0", "LimitNOFILE": "262144", "LimitNOFILESoft": "1024", "LimitNPROC": "127240", "LimitNPROCSoft": "127240", "LimitRSS": "infinity", "LimitRSSSoft": "infinity", "LimitRTPRIO": "0", "LimitRTPRIOSoft": "0", "LimitRTTIME": "infinity", "LimitRTTIMESoft": "infinity", "LimitSIGPENDING": "127240", "LimitSIGPENDINGSoft": "127240", "LimitSTACK": "infinity", "LimitSTACKSoft": "8388608", "LoadState": "loaded", "LockPersonality": "no", "LogLevelMax": "-1", "LogRateLimitBurst": "0", "LogRateLimitIntervalUSec": "0", "LogsDirectoryMode": "0755", "MainPID": "0", "MemoryAccounting": "yes", "MemoryCurrent": "[not set]", "MemoryDenyWriteExecute": "no", "MemoryHigh": "infinity", "MemoryLimit": "infinity", "MemoryLow": "0", "MemoryMax": "infinity", "MemoryMin": "0", "MemorySwapMax": "infinity", "MountAPIVFS": "no", "MountFlags": "", "NFileDescriptorStore": "0", "NRestarts": "0", "NUMAMask": "", "NUMAPolicy": "n/a", "Names": "firewalld.service", "NeedDaemonReload": "no", "Nice": "0", "NoNewPrivileges": "no", "NonBlocking": "no", "NotifyAccess": "none", "OOMScoreAdjust": "0", "OnFailureJobMode": "replace", "PermissionsStartOnly": "no", "Perpetual": "no", "PrivateDevices": "no", "PrivateMounts": "no", "PrivateNetwork": "no", "PrivateTmp": "no", "PrivateUsers": "no", "ProtectControlGroups": "no", "ProtectHome": "no", "ProtectKernelModules": "no", "ProtectKernelTunables": "no", "ProtectSystem": "no", "RefuseManualStart": "no", "RefuseManualStop": "no", "RemainAfterExit": "no", "RemoveIPC": "no", "Requires": "system.slice dbus.socket sysinit.target", "Restart": "no", "RestartUSec": "100ms", "RestrictNamespaces": "no", "RestrictRealtime": "no", "RestrictSUIDSGID": "no", "Result": "success", "RootDirectoryStartOnly": "no", "RuntimeDirectoryMode": "0755", "RuntimeDirectoryPreserve": "no", "RuntimeMaxUSec": "infinity", "SameProcessGroup": "no", "SecureBits": "0", "SendSIGHUP": "no", "SendSIGKILL": "yes", "Slice": "system.slice", "StandardError": "null", "StandardInput": "null 2021-11-11T20:44:47.124 INFO:teuthology.task.ansible.out:", "StandardInputData": "", "StandardOutput": "null", "StartLimitAction": "none", "StartLimitBurst": "5", "StartLimitIntervalUSec": "10s", "StartupBlockIOWeight": "[not set]", "StartupCPUShares": "[not set]", "StartupCPUWeight": "[not set]", "StartupIOWeight": "[not set]", "StateChangeTimestampMonotonic": "0", "StateDirectoryMode": "0755", "StatusErrno": "0", "StopWhenUnneeded": "no", "SubState": "dead", "SuccessAction": "none", "SyslogFacility": "3", "SyslogLevel": "6", "SyslogLevelPrefix": "yes", "SyslogPriority": "30", "SystemCallErrorNumber": "0", "TTYReset": "no", "TTYVHangup": "no", "TTYVTDisallocate": "no", "TasksAccounting": "yes", "TasksCurrent": "[not set]", "TasksMax": "203584", "TimeoutStartUSec": "1min 30s", "TimeoutStopUSec": "1min 30s", "TimerSlackNSec": "50000", "Transient": "no", "Type": "dbus", "UID": "[not set]", "UMask": "0022", "UnitFilePreset": "enabled", "UnitFileState": "disabled", "UtmpMode": "init", "Wants": "network-pre.target", "WatchdogTimestampMonotonic": "0", "WatchdogUSec": "0"}} 2021-11-11T20:44:47.141 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:47 +0000 (0:00:00.335) 0:07:07.397 ***** 2021-11-11T20:44:47.389 INFO:teuthology.task.ansible.out: TASK [testnode : Remove packages] ********************************************** 2021-11-11T20:44:47.391 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:44:47.415 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:47 +0000 (0:00:00.272) 0:07:07.670 ***** 2021-11-11T20:44:47.434 INFO:teuthology.task.ansible.out: TASK [testnode : Stop and disable iptables] ************************************ 2021-11-11T20:44:47.437 INFO:teuthology.task.ansible.out:fatal: [smithi096.front.sepia.ceph.com]: FAILED! => {"changed": false, "msg": "Could not find the requested service iptables: host"} ...ignoring 2021-11-11T20:44:47.459 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:47 +0000 (0:00:00.045) 0:07:07.715 ***** 2021-11-11T20:44:47.467 INFO:teuthology.task.ansible.out: TASK [testnode : Upgrade packages] ********************************************* 2021-11-11T20:44:47.469 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:47.497 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:47 +0000 (0:00:00.038) 0:07:07.753 ***** 2021-11-11T20:44:47.826 INFO:teuthology.task.ansible.out: TASK [testnode : Stop and disable firewalld] *********************************** 2021-11-11T20:44:47.827 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "enabled": false, "name": "firewalld", "state": "stopped", "status": {"ActiveEnterTimestampMonotonic": "0", "ActiveExitTimestampMonotonic": "0", "ActiveState": "inactive", "After": "system.slice dbus.socket polkit.service basic.target dbus.service sysinit.target", "AllowIsolate": "no", "AllowedCPUs": "", "AllowedMemoryNodes": "", "AmbientCapabilities": "", "AssertResult": "no", "AssertTimestampMonotonic": "0", "Before": "shutdown.target network-pre.target", "BlockIOAccounting": "no", "BlockIOWeight": "[not set]", "BusName": "org.fedoraproject.FirewallD1", "CPUAccounting": "no", "CPUAffinity": "", "CPUAffinityFromNUMA": "no", "CPUQuotaPerSecUSec": "infinity", "CPUQuotaPeriodUSec": "infinity", "CPUSchedulingPolicy": "0", "CPUSchedulingPriority": "0", "CPUSchedulingResetOnFork": "no", "CPUShares": "[not set]", "CPUUsageNSec": "[not set]", "CPUWeight": "[not set]", "CacheDirectoryMode": "0755", "CanFreeze": "yes", "CanIsolate": "no", "CanReload": "yes", "CanStart": "yes", "CanStop": "yes", "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon", "CollectMode": "inactive", "ConditionResult": "no", "ConditionTimestampMonotonic": "0", "ConfigurationDirectoryMode": "0755", "Conflicts": "nftables.service shutdown.target ebtables.service ipset.service ip6tables.service iptables.service", "ControlPID": "0", "DefaultDependencies": "yes", "DefaultMemoryLow": "0", "DefaultMemoryMin": "0", "Delegate": "no", 2021-11-11T20:44:47.829 INFO:teuthology.task.ansible.out:"Description": "firewalld - dynamic firewall daemon", "DevicePolicy": "auto", "Documentation": "man:firewalld(1)", "DynamicUser": "no", "EffectiveCPUs": "", "EffectiveMemoryNodes": "", "EnvironmentFiles": "/etc/sysconfig/firewalld (ignore_errors=yes)", "ExecMainCode": "0", "ExecMainExitTimestampMonotonic": "0", "ExecMainPID": "0", "ExecMainStartTimestampMonotonic": "0", "ExecMainStatus": "0", "ExecReload": "{ path=/bin/kill ; argv[]=/bin/kill -HUP $MAINPID ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "ExecStart": "{ path=/usr/sbin/firewalld ; argv[]=/usr/sbin/firewalld --nofork --nopid $FIREWALLD_ARGS ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "FailureAction": "none", "FileDescriptorStoreMax": "0", "FragmentPath": "/usr/lib/systemd/system/firewalld.service", "FreezerState": "running", "GID": "[not set]", "GuessMainPID": "yes", "IOAccounting": "no", "IOSchedulingClass": "0", "IOSchedulingPriority": "0", "IOWeight": "[not set]", "IPAccounting": "no", "IPEgressBytes": "18446744073709551615", "IPEgressPackets": "18446744073709551615", "IPIngressBytes": "18446744073709551615", "IPIngressPackets": "18446744073709551615", "Id": "firewalld.service", "IgnoreOnIsolate": "no", "IgnoreSIGPIPE": "yes", "InactiveEnterTimestampMonotonic": "0", "InactiveExitTimestampMonotonic": "0", "JobRunningTimeoutUSec": "infinity", "JobTimeoutAction": "none", "JobTimeoutUSec": "infinity", "KeyringMode": "private", "KillMode": "mixed", "KillSignal": "15", "LimitAS": "infinity", "LimitASSoft": "infinity", "LimitCORE": "infinity", "LimitCORESoft": "infinity", "LimitCPU": "infinity", "LimitCPUSoft": "infinity", "LimitDATA": "infinity", "LimitDATASoft": "infinity", "LimitFSIZE": "infinity", "LimitFSIZESoft": "infinity", "LimitLOCKS": "infinity", "LimitLOCKSSoft": "infinity", "LimitMEMLOCK": "65536", "LimitMEMLOCKSoft": "65536", "LimitMSGQUEUE": "819200", "LimitMSGQUEUESoft": "819200", "LimitNICE": " 2021-11-11T20:44:47.831 INFO:teuthology.task.ansible.out:0", "LimitNICESoft": "0", "LimitNOFILE": "262144", "LimitNOFILESoft": "1024", "LimitNPROC": "127240", "LimitNPROCSoft": "127240", "LimitRSS": "infinity", "LimitRSSSoft": "infinity", "LimitRTPRIO": "0", "LimitRTPRIOSoft": "0", "LimitRTTIME": "infinity", "LimitRTTIMESoft": "infinity", "LimitSIGPENDING": "127240", "LimitSIGPENDINGSoft": "127240", "LimitSTACK": "infinity", "LimitSTACKSoft": "8388608", "LoadState": "loaded", "LockPersonality": "no", "LogLevelMax": "-1", "LogRateLimitBurst": "0", "LogRateLimitIntervalUSec": "0", "LogsDirectoryMode": "0755", "MainPID": "0", "MemoryAccounting": "yes", "MemoryCurrent": "[not set]", "MemoryDenyWriteExecute": "no", "MemoryHigh": "infinity", "MemoryLimit": "infinity", "MemoryLow": "0", "MemoryMax": "infinity", "MemoryMin": "0", "MemorySwapMax": "infinity", "MountAPIVFS": "no", "MountFlags": "", "NFileDescriptorStore": "0", "NRestarts": "0", "NUMAMask": "", "NUMAPolicy": "n/a", "Names": "firewalld.service", "NeedDaemonReload": "no", "Nice": "0", "NoNewPrivileges": "no", "NonBlocking": "no", "NotifyAccess": "none", "OOMScoreAdjust": "0", "OnFailureJobMode": "replace", "PermissionsStartOnly": "no", "Perpetual": "no", "PrivateDevices": "no", "PrivateMounts": "no", "PrivateNetwork": "no", "PrivateTmp": "no", "PrivateUsers": "no", "ProtectControlGroups": "no", "ProtectHome": "no", "ProtectKernelModules": "no", "ProtectKernelTunables": "no", "ProtectSystem": "no", "RefuseManualStart": "no", "RefuseManualStop": "no", "RemainAfterExit": "no", "RemoveIPC": "no", "Requires": "sysinit.target dbus.socket system.slice", "Restart": "no", "RestartUSec": "100ms", "RestrictNamespaces": "no", "RestrictRealtime": "no", "RestrictSUIDSGID": "no", "Result": "success", "RootDirectoryStartOnly": "no", "RuntimeDirectoryMode": "0755", "RuntimeDirectoryPreserve": "no", "RuntimeMaxUSec": "infinity", "SameProcessGroup": "no", "SecureBits": "0", "SendSIGHUP": "no", "SendSIGKILL": "yes", "Slice": "system.slice", "StandardError": "null", "StandardInput": "null 2021-11-11T20:44:47.833 INFO:teuthology.task.ansible.out:", "StandardInputData": "", "StandardOutput": "null", "StartLimitAction": "none", "StartLimitBurst": "5", "StartLimitIntervalUSec": "10s", "StartupBlockIOWeight": "[not set]", "StartupCPUShares": "[not set]", "StartupCPUWeight": "[not set]", "StartupIOWeight": "[not set]", "StateChangeTimestampMonotonic": "0", "StateDirectoryMode": "0755", "StatusErrno": "0", "StopWhenUnneeded": "no", "SubState": "dead", "SuccessAction": "none", "SyslogFacility": "3", "SyslogLevel": "6", "SyslogLevelPrefix": "yes", "SyslogPriority": "30", "SystemCallErrorNumber": "0", "TTYReset": "no", "TTYVHangup": "no", "TTYVTDisallocate": "no", "TasksAccounting": "yes", "TasksCurrent": "[not set]", "TasksMax": "203584", "TimeoutStartUSec": "1min 30s", "TimeoutStopUSec": "1min 30s", "TimerSlackNSec": "50000", "Transient": "no", "Type": "dbus", "UID": "[not set]", "UMask": "0022", "UnitFilePreset": "enabled", "UnitFileState": "disabled", "UtmpMode": "init", "Wants": "network-pre.target", "WatchdogTimestampMonotonic": "0", "WatchdogUSec": "0"}} 2021-11-11T20:44:47.851 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:47 +0000 (0:00:00.353) 0:07:08.107 ***** 2021-11-11T20:44:48.125 INFO:teuthology.task.ansible.out: TASK [testnode : Enable SELinux] *********************************************** 2021-11-11T20:44:48.126 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "configfile": "/etc/selinux/config", "msg": "", "policy": "targeted", "reboot_required": false, "state": "permissive"} 2021-11-11T20:44:48.160 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.308) 0:07:08.415 ***** 2021-11-11T20:44:48.170 INFO:teuthology.task.ansible.out: TASK [testnode : Stop and disable iptables] ************************************ fatal: [smithi198.front.sepia.ceph.com]: FAILED! => {"changed": false, "msg": "Could not find the requested service iptables: host"} ...ignoring 2021-11-11T20:44:48.196 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.035) 0:07:08.451 ***** 2021-11-11T20:44:48.201 INFO:teuthology.task.ansible.out: TASK [testnode : Install abrt] ************************************************* 2021-11-11T20:44:48.203 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.232 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.037) 0:07:08.488 ***** 2021-11-11T20:44:48.261 INFO:teuthology.task.ansible.out: TASK [testnode : Enable abrt-auto-reporting] *********************************** 2021-11-11T20:44:48.263 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.287 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.054) 0:07:08.543 ***** 2021-11-11T20:44:48.321 INFO:teuthology.task.ansible.out: TASK [testnode : Set OpenGPGCheck in abrt-action-save-package-data.conf] ******* 2021-11-11T20:44:48.322 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.346 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.058) 0:07:08.602 ***** 2021-11-11T20:44:48.376 INFO:teuthology.task.ansible.out: TASK [testnode : Set ProcessUnpackaged in abrt-action-save-package-data.conf] *** 2021-11-11T20:44:48.378 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.401 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.054) 0:07:08.657 ***** 2021-11-11T20:44:48.441 INFO:teuthology.task.ansible.out: TASK [testnode : Restart abrtd] ************************************************ skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.466 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.064) 0:07:08.721 ***** 2021-11-11T20:44:48.476 INFO:teuthology.task.ansible.out: TASK [testnode : Enable SELinux] *********************************************** ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "configfile": "/etc/selinux/config", "msg": "", "policy": "targeted", "reboot_required": false, "state": "permissive"} 2021-11-11T20:44:48.500 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.034) 0:07:08.756 ***** 2021-11-11T20:44:48.507 INFO:teuthology.task.ansible.out: TASK [testnode : Check for custom repos] *************************************** 2021-11-11T20:44:48.509 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.532 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.031) 0:07:08.788 ***** 2021-11-11T20:44:48.538 INFO:teuthology.task.ansible.out: TASK [testnode : Install abrt] ************************************************* 2021-11-11T20:44:48.540 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.563 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.031) 0:07:08.819 ***** 2021-11-11T20:44:48.570 INFO:teuthology.task.ansible.out: TASK [testnode : Remove custom repos] ****************************************** 2021-11-11T20:44:48.594 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.031) 0:07:08.850 ***** 2021-11-11T20:44:48.613 INFO:teuthology.task.ansible.out: TASK [testnode : Enable abrt-auto-reporting] *********************************** 2021-11-11T20:44:48.615 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.639 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.044) 0:07:08.894 ***** 2021-11-11T20:44:48.648 INFO:teuthology.task.ansible.out: TASK [testnode : Set apt preferences] ****************************************** 2021-11-11T20:44:48.650 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.681 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.042) 0:07:08.937 ***** 2021-11-11T20:44:48.692 INFO:teuthology.task.ansible.out: TASK [testnode : Set OpenGPGCheck in abrt-action-save-package-data.conf] ******* 2021-11-11T20:44:48.694 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.717 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.035) 0:07:08.972 ***** 2021-11-11T20:44:48.725 INFO:teuthology.task.ansible.out: TASK [testnode : Add sources list] ********************************************* 2021-11-11T20:44:48.727 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.753 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.035) 0:07:09.008 ***** 2021-11-11T20:44:48.760 INFO:teuthology.task.ansible.out: TASK [testnode : Set ProcessUnpackaged in abrt-action-save-package-data.conf] *** 2021-11-11T20:44:48.762 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.786 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.033) 0:07:09.042 ***** 2021-11-11T20:44:48.795 INFO:teuthology.task.ansible.out: TASK [testnode : Install apt keys] ********************************************* 2021-11-11T20:44:48.796 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=http://download.ceph.com/keys/autobuild.asc) => {"ansible_loop_var": "item", "changed": false, "item": "http://download.ceph.com/keys/autobuild.asc", "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.796 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=http://download.ceph.com/keys/release.asc) => {"ansible_loop_var": "item", "changed": false, "item": "http://download.ceph.com/keys/release.asc", "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.831 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.045) 0:07:09.087 ***** 2021-11-11T20:44:48.838 INFO:teuthology.task.ansible.out: TASK [testnode : Restart abrtd] ************************************************ 2021-11-11T20:44:48.840 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.865 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.033) 0:07:09.121 ***** 2021-11-11T20:44:48.871 INFO:teuthology.task.ansible.out: TASK [testnode : Install python-apt] ******************************************* 2021-11-11T20:44:48.873 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.896 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.030) 0:07:09.151 ***** 2021-11-11T20:44:48.903 INFO:teuthology.task.ansible.out: TASK [testnode : Check for custom repos] *************************************** skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:48.927 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.031) 0:07:09.183 ***** 2021-11-11T20:44:48.935 INFO:teuthology.task.ansible.out: TASK [testnode : Add local apt repos.] ***************************************** 2021-11-11T20:44:48.959 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.032) 0:07:09.215 ***** 2021-11-11T20:44:48.966 INFO:teuthology.task.ansible.out: TASK [testnode : Remove custom repos] ****************************************** 2021-11-11T20:44:48.996 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:48 +0000 (0:00:00.036) 0:07:09.252 ***** 2021-11-11T20:44:49.003 INFO:teuthology.task.ansible.out: TASK [testnode : Blast the apt cache.] ***************************************** 2021-11-11T20:44:49.005 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.051 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.051) 0:07:09.303 ***** 2021-11-11T20:44:49.058 INFO:teuthology.task.ansible.out: TASK [testnode : Set apt preferences] ****************************************** skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.085 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.037) 0:07:09.341 ***** 2021-11-11T20:44:49.094 INFO:teuthology.task.ansible.out: TASK [testnode : Update apt cache.] ******************************************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.123 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.037) 0:07:09.379 ***** 2021-11-11T20:44:49.133 INFO:teuthology.task.ansible.out: TASK [testnode : Add sources list] ********************************************* 2021-11-11T20:44:49.135 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.159 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.035) 0:07:09.415 ***** 2021-11-11T20:44:49.167 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure packages are not present.] ***************************** 2021-11-11T20:44:49.169 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.192 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.033) 0:07:09.448 ***** 2021-11-11T20:44:49.210 INFO:teuthology.task.ansible.out: TASK [testnode : Install apt keys] ********************************************* 2021-11-11T20:44:49.212 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=http://download.ceph.com/keys/autobuild.asc) => {"ansible_loop_var": "item", "changed": false, "item": "http://download.ceph.com/keys/autobuild.asc", "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item=http://download.ceph.com/keys/release.asc) => {"ansible_loop_var": "item", "changed": false, "item": "http://download.ceph.com/keys/release.asc", "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.242 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.049) 0:07:09.498 ***** 2021-11-11T20:44:49.252 INFO:teuthology.task.ansible.out: TASK [testnode : Upgrade packages] ********************************************* 2021-11-11T20:44:49.254 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.291 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.049) 0:07:09.547 ***** 2021-11-11T20:44:49.302 INFO:teuthology.task.ansible.out: TASK [testnode : Install python-apt] ******************************************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.333 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.041) 0:07:09.589 ***** 2021-11-11T20:44:49.345 INFO:teuthology.task.ansible.out: TASK [testnode : Upgrade non aarch64 packages] ********************************* skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.372 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.039) 0:07:09.628 ***** 2021-11-11T20:44:49.379 INFO:teuthology.task.ansible.out: TASK [testnode : Add local apt repos.] ***************************************** 2021-11-11T20:44:49.405 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.033) 0:07:09.661 ***** 2021-11-11T20:44:49.413 INFO:teuthology.task.ansible.out: TASK [testnode : Install packages] ********************************************* 2021-11-11T20:44:49.416 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.441 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.035) 0:07:09.697 ***** 2021-11-11T20:44:49.448 INFO:teuthology.task.ansible.out: TASK [testnode : Blast the apt cache.] ***************************************** 2021-11-11T20:44:49.450 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.474 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.033) 0:07:09.730 ***** 2021-11-11T20:44:49.482 INFO:teuthology.task.ansible.out: TASK [testnode : Install non aarch64 packages] ********************************* 2021-11-11T20:44:49.483 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.507 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.032) 0:07:09.763 ***** 2021-11-11T20:44:49.515 INFO:teuthology.task.ansible.out: TASK [testnode : Update apt cache.] ******************************************** 2021-11-11T20:44:49.517 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.540 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.033) 0:07:09.796 ***** 2021-11-11T20:44:49.548 INFO:teuthology.task.ansible.out: TASK [testnode : Install packages with --no-install-recommends] **************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.573 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.032) 0:07:09.829 ***** 2021-11-11T20:44:49.582 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure packages are not present.] ***************************** 2021-11-11T20:44:49.583 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.615 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.041) 0:07:09.871 ***** 2021-11-11T20:44:49.625 INFO:teuthology.task.ansible.out: TASK [testnode : Use perl to add user_xattr to the root mount options in fstab.] *** 2021-11-11T20:44:49.628 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.658 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.042) 0:07:09.914 ***** 2021-11-11T20:44:49.665 INFO:teuthology.task.ansible.out: TASK [testnode : Upgrade packages] ********************************************* 2021-11-11T20:44:49.667 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.691 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.033) 0:07:09.947 ***** 2021-11-11T20:44:49.699 INFO:teuthology.task.ansible.out: TASK [testnode : Enable xattr for this boot.] ********************************** 2021-11-11T20:44:49.701 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.725 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.032) 0:07:09.980 ***** 2021-11-11T20:44:49.732 INFO:teuthology.task.ansible.out: TASK [testnode : Upgrade non aarch64 packages] ********************************* 2021-11-11T20:44:49.734 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.760 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.035) 0:07:10.016 ***** 2021-11-11T20:44:49.767 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure fuse group exists.] ************************************ 2021-11-11T20:44:49.769 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.791 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.031) 0:07:10.047 ***** 2021-11-11T20:44:49.799 INFO:teuthology.task.ansible.out: TASK [testnode : Install packages] ********************************************* 2021-11-11T20:44:49.801 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.823 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.032) 0:07:10.079 ***** 2021-11-11T20:44:49.832 INFO:teuthology.task.ansible.out: TASK [testnode : Upload /etc/fuse.conf.] *************************************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.856 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.032) 0:07:10.112 ***** 2021-11-11T20:44:49.863 INFO:teuthology.task.ansible.out: TASK [testnode : Install non aarch64 packages] ********************************* 2021-11-11T20:44:49.864 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.889 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.033) 0:07:10.145 ***** 2021-11-11T20:44:49.897 INFO:teuthology.task.ansible.out: TASK [testnode : Add teuthology user to groups fuse, kvm and disk.] ************ 2021-11-11T20:44:49.898 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.922 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.033) 0:07:10.178 ***** 2021-11-11T20:44:49.930 INFO:teuthology.task.ansible.out: TASK [testnode : Install packages with --no-install-recommends] **************** skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.955 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.032) 0:07:10.211 ***** 2021-11-11T20:44:49.962 INFO:teuthology.task.ansible.out: TASK [testnode : Set up static IP in /etc/hosts.] ****************************** 2021-11-11T20:44:49.964 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:49.989 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:49 +0000 (0:00:00.033) 0:07:10.244 ***** 2021-11-11T20:44:49.997 INFO:teuthology.task.ansible.out: TASK [testnode : Use perl to add user_xattr to the root mount options in fstab.] *** 2021-11-11T20:44:49.998 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.023 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.034) 0:07:10.279 ***** 2021-11-11T20:44:50.031 INFO:teuthology.task.ansible.out: TASK [testnode : Stop apache2] ************************************************* skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.057 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.033) 0:07:10.313 ***** 2021-11-11T20:44:50.065 INFO:teuthology.task.ansible.out: TASK [testnode : Enable xattr for this boot.] ********************************** 2021-11-11T20:44:50.068 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.090 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.032) 0:07:10.346 ***** 2021-11-11T20:44:50.098 INFO:teuthology.task.ansible.out: TASK [testnode : Set mode on /etc/fuse.conf] *********************************** 2021-11-11T20:44:50.100 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.122 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.032) 0:07:10.378 ***** 2021-11-11T20:44:50.131 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure fuse group exists.] ************************************ 2021-11-11T20:44:50.133 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.156 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.033) 0:07:10.412 ***** 2021-11-11T20:44:50.164 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure the group kvm exists.] ********************************* 2021-11-11T20:44:50.166 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.188 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.031) 0:07:10.444 ***** 2021-11-11T20:44:50.195 INFO:teuthology.task.ansible.out: TASK [testnode : Upload /etc/fuse.conf.] *************************************** 2021-11-11T20:44:50.197 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.220 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.032) 0:07:10.476 ***** 2021-11-11T20:44:50.227 INFO:teuthology.task.ansible.out: TASK [testnode : Add the teuthology user to groups kvm,disk] ******************* 2021-11-11T20:44:50.229 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.259 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.031) 0:07:10.508 ***** 2021-11-11T20:44:50.269 INFO:teuthology.task.ansible.out: TASK [testnode : Add teuthology user to groups fuse, kvm and disk.] ************ 2021-11-11T20:44:50.271 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.294 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.042) 0:07:10.550 ***** 2021-11-11T20:44:50.302 INFO:teuthology.task.ansible.out: TASK [testnode : Configure /etc/sudoers.] ************************************** 2021-11-11T20:44:50.304 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.328 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.034) 0:07:10.584 ***** 2021-11-11T20:44:50.336 INFO:teuthology.task.ansible.out: TASK [testnode : Set up static IP in /etc/hosts.] ****************************** 2021-11-11T20:44:50.338 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.361 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.032) 0:07:10.617 ***** 2021-11-11T20:44:50.369 INFO:teuthology.task.ansible.out: TASK [testnode : Configure /etc/security/limits.conf] ************************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.401 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.039) 0:07:10.656 ***** 2021-11-11T20:44:50.407 INFO:teuthology.task.ansible.out: TASK [testnode : Stop apache2] ************************************************* 2021-11-11T20:44:50.409 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.433 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.033) 0:07:10.689 ***** 2021-11-11T20:44:50.443 INFO:teuthology.task.ansible.out: TASK [testnode : Rebuild rpmdb] ************************************************ 2021-11-11T20:44:50.444 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.468 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.033) 0:07:10.723 ***** 2021-11-11T20:44:50.476 INFO:teuthology.task.ansible.out: TASK [testnode : Set mode on /etc/fuse.conf] *********************************** 2021-11-11T20:44:50.478 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.501 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.033) 0:07:10.756 ***** 2021-11-11T20:44:50.509 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure ceph packages are not present.] ************************ 2021-11-11T20:44:50.511 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.534 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.033) 0:07:10.790 ***** 2021-11-11T20:44:50.542 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure the group kvm exists.] ********************************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.573 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.039) 0:07:10.829 ***** 2021-11-11T20:44:50.583 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure ceph dependency packages are not present.] ************* skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.609 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.034) 0:07:10.865 ***** 2021-11-11T20:44:50.618 INFO:teuthology.task.ansible.out: TASK [testnode : Add the teuthology user to groups kvm,disk] ******************* 2021-11-11T20:44:50.620 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.643 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.034) 0:07:10.899 ***** 2021-11-11T20:44:50.652 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** 2021-11-11T20:44:50.654 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.679 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.034) 0:07:10.933 ***** 2021-11-11T20:44:50.685 INFO:teuthology.task.ansible.out: TASK [testnode : Configure /etc/sudoers.] ************************************** 2021-11-11T20:44:50.686 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.715 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.037) 0:07:10.971 ***** 2021-11-11T20:44:50.725 INFO:teuthology.task.ansible.out: TASK [testnode : Remove packages] ********************************************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.751 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.036) 0:07:11.007 ***** 2021-11-11T20:44:50.759 INFO:teuthology.task.ansible.out: TASK [testnode : Configure /etc/security/limits.conf] ************************** 2021-11-11T20:44:50.761 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.784 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.032) 0:07:11.040 ***** 2021-11-11T20:44:50.791 INFO:teuthology.task.ansible.out: TASK [testnode : Install packages] ********************************************* 2021-11-11T20:44:50.793 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.814 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.030) 0:07:11.070 ***** 2021-11-11T20:44:50.821 INFO:teuthology.task.ansible.out: TASK [testnode : Rebuild rpmdb] ************************************************ 2021-11-11T20:44:50.823 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.848 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.034) 0:07:11.104 ***** 2021-11-11T20:44:50.855 INFO:teuthology.task.ansible.out: TASK [testnode : Upgrade packages] ********************************************* 2021-11-11T20:44:50.857 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.880 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.032) 0:07:11.136 ***** 2021-11-11T20:44:50.888 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure ceph packages are not present.] ************************ 2021-11-11T20:44:50.889 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.913 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.032) 0:07:11.169 ***** 2021-11-11T20:44:50.920 INFO:teuthology.task.ansible.out: TASK [testnode : Include hostname in /etc/sysconfig/network] ******************* 2021-11-11T20:44:50.922 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.944 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.031) 0:07:11.200 ***** 2021-11-11T20:44:50.952 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure ceph dependency packages are not present.] ************* 2021-11-11T20:44:50.953 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:50.984 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:50 +0000 (0:00:00.039) 0:07:11.240 ***** 2021-11-11T20:44:50.994 INFO:teuthology.task.ansible.out: TASK [testnode : Make raid/smart scripts work.] ******************************** 2021-11-11T20:44:50.996 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.018 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.034) 0:07:11.274 ***** 2021-11-11T20:44:51.026 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** 2021-11-11T20:44:51.028 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.050 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.032) 0:07:11.306 ***** 2021-11-11T20:44:51.058 INFO:teuthology.task.ansible.out: TASK [testnode : Create FStest ubuntu directory.] ****************************** 2021-11-11T20:44:51.060 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.082 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.031) 0:07:11.338 ***** 2021-11-11T20:44:51.090 INFO:teuthology.task.ansible.out: TASK [testnode : Remove packages] ********************************************** 2021-11-11T20:44:51.092 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.116 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.034) 0:07:11.372 ***** 2021-11-11T20:44:51.125 INFO:teuthology.task.ansible.out: TASK [testnode : Make fsstress same path as ubuntu.] *************************** 2021-11-11T20:44:51.126 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.152 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.035) 0:07:11.408 ***** 2021-11-11T20:44:51.161 INFO:teuthology.task.ansible.out: TASK [testnode : Install packages] ********************************************* 2021-11-11T20:44:51.163 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.186 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.033) 0:07:11.442 ***** 2021-11-11T20:44:51.194 INFO:teuthology.task.ansible.out: TASK [testnode : Upload a dummy nfs export so that the nfs kernel server starts.] *** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.219 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.032) 0:07:11.475 ***** 2021-11-11T20:44:51.227 INFO:teuthology.task.ansible.out: TASK [testnode : Upgrade packages] ********************************************* 2021-11-11T20:44:51.229 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.254 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.034) 0:07:11.509 ***** 2021-11-11T20:44:51.273 INFO:teuthology.task.ansible.out: TASK [testnode : Enable nfs-server on rhel 7.x.] ******************************* 2021-11-11T20:44:51.281 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.301 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.047) 0:07:11.557 ***** 2021-11-11T20:44:51.310 INFO:teuthology.task.ansible.out: TASK [testnode : Include hostname in /etc/sysconfig/network] ******************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.336 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.034) 0:07:11.591 ***** 2021-11-11T20:44:51.342 INFO:teuthology.task.ansible.out: TASK [testnode : Include hostname in /etc/sysconfig/network] ******************* 2021-11-11T20:44:51.344 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.367 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.031) 0:07:11.623 ***** 2021-11-11T20:44:51.375 INFO:teuthology.task.ansible.out: TASK [testnode : Make raid/smart scripts work.] ******************************** 2021-11-11T20:44:51.376 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.405 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.037) 0:07:11.660 ***** 2021-11-11T20:44:51.413 INFO:teuthology.task.ansible.out: TASK [testnode : Make raid/smart scripts work.] ******************************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.439 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.034) 0:07:11.695 ***** 2021-11-11T20:44:51.446 INFO:teuthology.task.ansible.out: TASK [testnode : Create FStest ubuntu directory.] ****************************** 2021-11-11T20:44:51.451 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.474 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.033) 0:07:11.728 ***** 2021-11-11T20:44:51.480 INFO:teuthology.task.ansible.out: TASK [testnode : Create FStest ubuntu directory.] ****************************** 2021-11-11T20:44:51.492 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.506 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.033) 0:07:11.761 ***** 2021-11-11T20:44:51.514 INFO:teuthology.task.ansible.out: TASK [testnode : Make fsstress same path as ubuntu.] *************************** 2021-11-11T20:44:51.516 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.540 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.033) 0:07:11.795 ***** 2021-11-11T20:44:51.547 INFO:teuthology.task.ansible.out: TASK [testnode : Make fsstress same path as ubuntu.] *************************** 2021-11-11T20:44:51.551 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.573 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.034) 0:07:11.829 ***** 2021-11-11T20:44:51.582 INFO:teuthology.task.ansible.out: TASK [testnode : Upload a dummy nfs export so that the nfs kernel server starts.] *** skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.607 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.034) 0:07:11.863 ***** 2021-11-11T20:44:51.615 INFO:teuthology.task.ansible.out: TASK [testnode : Make raid/smart scripts work.] ******************************** 2021-11-11T20:44:51.617 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.644 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.035) 0:07:11.899 ***** 2021-11-11T20:44:51.651 INFO:teuthology.task.ansible.out: TASK [testnode : Enable nfs-server on rhel 7.x.] ******************************* 2021-11-11T20:44:51.653 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.679 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.034) 0:07:11.934 ***** 2021-11-11T20:44:51.689 INFO:teuthology.task.ansible.out: TASK [testnode : Create FStest ubuntu directory.] ****************************** 2021-11-11T20:44:51.691 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.715 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.036) 0:07:11.971 ***** 2021-11-11T20:44:51.724 INFO:teuthology.task.ansible.out: TASK [testnode : Include hostname in /etc/sysconfig/network] ******************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.749 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.034) 0:07:12.005 ***** 2021-11-11T20:44:51.758 INFO:teuthology.task.ansible.out: TASK [testnode : Make fsstress same path as ubuntu.] *************************** 2021-11-11T20:44:51.759 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.783 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.033) 0:07:12.039 ***** 2021-11-11T20:44:51.792 INFO:teuthology.task.ansible.out: TASK [testnode : Make raid/smart scripts work.] ******************************** skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.817 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.033) 0:07:12.073 ***** 2021-11-11T20:44:51.825 INFO:teuthology.task.ansible.out: TASK [testnode : Set grub config.] ********************************************* 2021-11-11T20:44:51.827 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.855 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.038) 0:07:12.111 ***** 2021-11-11T20:44:51.865 INFO:teuthology.task.ansible.out: TASK [testnode : Create FStest ubuntu directory.] ****************************** 2021-11-11T20:44:51.867 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.890 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.035) 0:07:12.146 ***** 2021-11-11T20:44:51.908 INFO:teuthology.task.ansible.out: TASK [testnode : Remove /etc/ceph] ********************************************* 2021-11-11T20:44:51.910 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.930 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.040) 0:07:12.186 ***** 2021-11-11T20:44:51.940 INFO:teuthology.task.ansible.out: TASK [testnode : Make fsstress same path as ubuntu.] *************************** 2021-11-11T20:44:51.942 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:51.968 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.036) 0:07:12.223 ***** 2021-11-11T20:44:51.974 INFO:teuthology.task.ansible.out: TASK [testnode : Upload a dummy nfs export so that the nfs kernel server starts.] *** 2021-11-11T20:44:51.976 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.000 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:51 +0000 (0:00:00.032) 0:07:12.255 ***** 2021-11-11T20:44:52.008 INFO:teuthology.task.ansible.out: TASK [testnode : Make raid/smart scripts work.] ******************************** 2021-11-11T20:44:52.010 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.035 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.035) 0:07:12.291 ***** 2021-11-11T20:44:52.043 INFO:teuthology.task.ansible.out: TASK [testnode : Enable nfs-server on rhel 7.x.] ******************************* 2021-11-11T20:44:52.045 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.068 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.032) 0:07:12.323 ***** 2021-11-11T20:44:52.074 INFO:teuthology.task.ansible.out: TASK [testnode : Create FStest ubuntu directory.] ****************************** 2021-11-11T20:44:52.077 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.099 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.031) 0:07:12.354 ***** 2021-11-11T20:44:52.106 INFO:teuthology.task.ansible.out: TASK [testnode : Upload weekly kernel-clean crontab.] ************************** 2021-11-11T20:44:52.108 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.131 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.032) 0:07:12.387 ***** 2021-11-11T20:44:52.138 INFO:teuthology.task.ansible.out: TASK [testnode : Make fsstress same path as ubuntu.] *************************** 2021-11-11T20:44:52.140 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.163 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.032) 0:07:12.419 ***** 2021-11-11T20:44:52.171 INFO:teuthology.task.ansible.out: TASK [testnode : Upload /etc/grub.d/02_force_timeout.] ************************* skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.194 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.030) 0:07:12.450 ***** 2021-11-11T20:44:52.201 INFO:teuthology.task.ansible.out: TASK [testnode : Set grub config.] ********************************************* 2021-11-11T20:44:52.203 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.229 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.034) 0:07:12.485 ***** 2021-11-11T20:44:52.236 INFO:teuthology.task.ansible.out: TASK [testnode : Enable kernel modules to load at boot time.] ****************** 2021-11-11T20:44:52.238 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.262 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.032) 0:07:12.518 ***** 2021-11-11T20:44:52.269 INFO:teuthology.task.ansible.out: TASK [testnode : Remove /etc/ceph] ********************************************* 2021-11-11T20:44:52.271 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.293 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.030) 0:07:12.549 ***** 2021-11-11T20:44:52.300 INFO:teuthology.task.ansible.out: TASK [testnode : Enabling auto-fsck fix to prevent boot hangup.] *************** 2021-11-11T20:44:52.302 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.324 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.031) 0:07:12.580 ***** 2021-11-11T20:44:52.332 INFO:teuthology.task.ansible.out: TASK [testnode : Upload a dummy nfs export so that the nfs kernel server starts.] *** 2021-11-11T20:44:52.334 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.356 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.032) 0:07:12.612 ***** 2021-11-11T20:44:52.364 INFO:teuthology.task.ansible.out: TASK [testnode : Work around broken wget on wheezy.] *************************** 2021-11-11T20:44:52.365 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.388 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.031) 0:07:12.644 ***** 2021-11-11T20:44:52.395 INFO:teuthology.task.ansible.out: TASK [testnode : Enable nfs-server on rhel 7.x.] ******************************* 2021-11-11T20:44:52.397 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.419 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.031) 0:07:12.675 ***** 2021-11-11T20:44:52.426 INFO:teuthology.task.ansible.out: TASK [testnode : Stop collectl] ************************************************ 2021-11-11T20:44:52.427 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.453 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.034) 0:07:12.709 ***** 2021-11-11T20:44:52.461 INFO:teuthology.task.ansible.out: TASK [testnode : Upload weekly kernel-clean crontab.] ************************** 2021-11-11T20:44:52.463 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.486 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.032) 0:07:12.742 ***** 2021-11-11T20:44:52.505 INFO:teuthology.task.ansible.out: TASK [testnode : Add PATH to the teuthology_user .bashrc.] ********************* skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.526 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.039) 0:07:12.782 ***** 2021-11-11T20:44:52.533 INFO:teuthology.task.ansible.out: TASK [testnode : Upload /etc/grub.d/02_force_timeout.] ************************* 2021-11-11T20:44:52.535 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.558 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.031) 0:07:12.814 ***** 2021-11-11T20:44:52.566 INFO:teuthology.task.ansible.out: TASK [testnode : Check to see if we need to edit /etc/profile.] **************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.600 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.042) 0:07:12.856 ***** 2021-11-11T20:44:52.609 INFO:teuthology.task.ansible.out: TASK [testnode : Enable kernel modules to load at boot time.] ****************** 2021-11-11T20:44:52.610 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.636 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.036) 0:07:12.892 ***** 2021-11-11T20:44:52.645 INFO:teuthology.task.ansible.out: TASK [testnode : Update /etc/profile if needed.] ******************************* 2021-11-11T20:44:52.647 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.669 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.032) 0:07:12.925 ***** 2021-11-11T20:44:52.676 INFO:teuthology.task.ansible.out: TASK [testnode : Enabling auto-fsck fix to prevent boot hangup.] *************** 2021-11-11T20:44:52.678 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.709 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.036) 0:07:12.962 ***** 2021-11-11T20:44:52.718 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** 2021-11-11T20:44:52.720 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"nvme_card_present": false}, "changed": false} 2021-11-11T20:44:52.751 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.045) 0:07:13.007 ***** 2021-11-11T20:44:52.759 INFO:teuthology.task.ansible.out: TASK [testnode : Work around broken wget on wheezy.] *************************** 2021-11-11T20:44:52.761 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.790 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.038) 0:07:13.046 ***** 2021-11-11T20:44:52.808 INFO:teuthology.task.ansible.out: TASK [testnode : Check for NVMe drive] ***************************************** 2021-11-11T20:44:52.810 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=dm-1) => {"ansible_loop_var": "item", "changed": false, "item": "dm-1", "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.819 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=nvme0n1) => {"ansible_facts": {"nvme_card_present": true}, "ansible_loop_var": "item", "changed": false, "item": "nvme0n1"} 2021-11-11T20:44:52.821 INFO:teuthology.task.ansible.out: TASK [testnode : Stop collectl] ************************************************ skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.846 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.056) 0:07:13.102 ***** 2021-11-11T20:44:52.856 INFO:teuthology.task.ansible.out: TASK [testnode : Check for NVMe drive] ***************************************** 2021-11-11T20:44:52.858 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=dm-4) => {"ansible_loop_var": "item", "changed": false, "item": "dm-4", "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item=dm-2) => {"ansible_loop_var": "item", "changed": false, "item": "dm-2", "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item=dm-0) => {"ansible_loop_var": "item", "changed": false, "item": "dm-0", "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item=sda) => {"ansible_loop_var": "item", "changed": false, "item": "sda", "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item=dm-3) => {"ansible_loop_var": "item", "changed": false, "item": "dm-3", "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.885 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.038) 0:07:13.141 ***** 2021-11-11T20:44:52.892 INFO:teuthology.task.ansible.out: TASK [testnode : Add PATH to the teuthology_user .bashrc.] ********************* 2021-11-11T20:44:52.894 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.919 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.033) 0:07:13.175 ***** 2021-11-11T20:44:52.959 INFO:teuthology.task.ansible.out: TASK [testnode : Check to see if we need to edit /etc/profile.] **************** skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:52.981 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:52 +0000 (0:00:00.061) 0:07:13.237 ***** 2021-11-11T20:44:53.021 INFO:teuthology.task.ansible.out: TASK [testnode : Update /etc/profile if needed.] ******************************* 2021-11-11T20:44:53.022 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.053 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.072) 0:07:13.309 ***** 2021-11-11T20:44:53.086 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"nvme_card_present": false}, "changed": false} 2021-11-11T20:44:53.113 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.059) 0:07:13.369 ***** 2021-11-11T20:44:53.132 INFO:teuthology.task.ansible.out: TASK [testnode : Check for teuthology-lock command] **************************** 2021-11-11T20:44:53.133 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": "which teuthology-lock", "delta": "0:00:00.003286", "end": "2021-11-11 20:44:53.104961", "rc": 0, "start": "2021-11-11 20:44:53.101675", "stderr": "", "stderr_lines": [], "stdout": "/home/teuthworker/src/git.ceph.com_git_teuthology_d4737010a85099043cf081dc05b4069d301b23fb/virtualenv/bin/teuthology-lock", "stdout_lines": ["/home/teuthworker/src/git.ceph.com_git_teuthology_d4737010a85099043cf081dc05b4069d301b23fb/virtualenv/bin/teuthology-lock"]} 2021-11-11T20:44:53.162 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.048) 0:07:13.418 ***** 2021-11-11T20:44:53.170 INFO:teuthology.task.ansible.out: TASK [testnode : Check for NVMe drive] ***************************************** 2021-11-11T20:44:53.172 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=dm-1) => {"ansible_loop_var": "item", "changed": false, "item": "dm-1", "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.185 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=nvme0n1) => {"ansible_facts": {"nvme_card_present": true}, "ansible_loop_var": "item", "changed": false, "item": "nvme0n1"} skipping: [smithi198.front.sepia.ceph.com] => (item=dm-4) => {"ansible_loop_var": "item", "changed": false, "item": "dm-4", "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item=dm-2) => {"ansible_loop_var": "item", "changed": false, "item": "dm-2", "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.187 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=dm-0) => {"ansible_loop_var": "item", "changed": false, "item": "dm-0", "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.196 INFO:teuthology.task.ansible.out: TASK [testnode : Mark system down if NVMe card missing] ************************ 2021-11-11T20:44:53.198 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} TASK [testnode : Check for NVMe drive] ***************************************** skipping: [smithi198.front.sepia.ceph.com] => (item=sda) => {"ansible_loop_var": "item", "changed": false, "item": "sda", "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item=dm-3) => {"ansible_loop_var": "item", "changed": false, "item": "dm-3", "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.232 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.070) 0:07:13.488 ***** 2021-11-11T20:44:53.261 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.029) 0:07:13.517 ***** 2021-11-11T20:44:53.273 INFO:teuthology.task.ansible.out: TASK [testnode : Update description in paddles lock DB if NVMe card missing] *** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.296 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.034) 0:07:13.552 ***** 2021-11-11T20:44:53.336 INFO:teuthology.task.ansible.out: TASK [testnode : Fail rest of playbook due to missing NVMe card] *************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.361 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.065) 0:07:13.617 ***** 2021-11-11T20:44:53.391 INFO:teuthology.task.ansible.out: TASK [testnode : Make sure apt dependencies are installed] ********************* 2021-11-11T20:44:53.393 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.414 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.052) 0:07:13.670 ***** 2021-11-11T20:44:53.465 INFO:teuthology.task.ansible.out: TASK [testnode : Check for teuthology-lock command] **************************** 2021-11-11T20:44:53.467 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": "which teuthology-lock", "delta": "0:00:00.003525", "end": "2021-11-11 20:44:53.439854", "rc": 0, "start": "2021-11-11 20:44:53.436329", "stderr": "", "stderr_lines": [], "stdout": "/home/teuthworker/src/git.ceph.com_git_teuthology_d4737010a85099043cf081dc05b4069d301b23fb/virtualenv/bin/teuthology-lock", "stdout_lines": ["/home/teuthworker/src/git.ceph.com_git_teuthology_d4737010a85099043cf081dc05b4069d301b23fb/virtualenv/bin/teuthology-lock"]} 2021-11-11T20:44:53.494 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.079) 0:07:13.750 ***** 2021-11-11T20:44:53.522 INFO:teuthology.task.ansible.out: TASK [testnode : Mark system down if NVMe card missing] ************************ 2021-11-11T20:44:53.524 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.551 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.057) 0:07:13.807 ***** 2021-11-11T20:44:53.594 INFO:teuthology.task.ansible.out: TASK [testnode : Update description in paddles lock DB if NVMe card missing] *** 2021-11-11T20:44:53.596 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.623 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.072) 0:07:13.879 ***** 2021-11-11T20:44:53.653 INFO:teuthology.task.ansible.out: TASK [testnode : Fail rest of playbook due to missing NVMe card] *************** 2021-11-11T20:44:53.655 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.686 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.062) 0:07:13.942 ***** 2021-11-11T20:44:53.717 INFO:teuthology.task.ansible.out: TASK [testnode : Make sure apt dependencies are installed] ********************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:53.745 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:53 +0000 (0:00:00.058) 0:07:14.001 ***** 2021-11-11T20:44:56.461 INFO:teuthology.task.ansible.out: TASK [testnode : Make sure rpm dependencies are installed] ********************* 2021-11-11T20:44:56.463 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:44:56.484 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:56 +0000 (0:00:02.738) 0:07:16.739 ***** 2021-11-11T20:44:56.522 INFO:teuthology.task.ansible.out: TASK [testnode : Set root disk] ************************************************ 2021-11-11T20:44:56.524 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item={'mount': '/', 'device': '/dev/sda1', 'fstype': 'ext4', 'options': 'rw,seclabel,relatime', 'size_total': 983429562368, 'size_available': 930320162816, 'block_size': 4096, 'block_total': 240095108, 'block_available': 227128946, 'block_used': 12966162, 'inode_total': 61054976, 'inode_available': 60985436, 'inode_used': 69540, 'uuid': 'bc802139-74bd-43cf-840d-e30da2bb57f8'}) => {"ansible_facts": {"root_disk": "/dev/sda"}, "ansible_loop_var": "item", "changed": false, "item": {"block_available": 227128946, "block_size": 4096, "block_total": 240095108, "block_used": 12966162, "device": "/dev/sda1", "fstype": "ext4", "inode_available": 60985436, "inode_total": 61054976, "inode_used": 69540, "mount": "/", "options": "rw,seclabel,relatime", "size_available": 930320162816, "size_total": 983429562368, "uuid": "bc802139-74bd-43cf-840d-e30da2bb57f8"}} 2021-11-11T20:44:56.546 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:56 +0000 (0:00:00.062) 0:07:16.802 ***** 2021-11-11T20:44:56.731 INFO:teuthology.task.ansible.out: TASK [testnode : Make sure rpm dependencies are installed] ********************* ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:44:56.756 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:56 +0000 (0:00:00.209) 0:07:17.012 ***** 2021-11-11T20:44:56.763 INFO:teuthology.task.ansible.out: TASK [testnode : Compile list of non-root partitions] ************************** 2021-11-11T20:44:56.765 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": "lsblk --list --noheadings | grep part | grep -v sda | awk '{ print $1 }'", "delta": "0:00:00.005196", "end": "2021-11-11 20:44:56.721002", "rc": 0, "start": "2021-11-11 20:44:56.715806", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:44:56.790 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:56 +0000 (0:00:00.034) 0:07:17.046 ***** 2021-11-11T20:44:56.810 INFO:teuthology.task.ansible.out: TASK [testnode : Set root disk] ************************************************ 2021-11-11T20:44:56.812 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item={'mount': '/', 'device': '/dev/sda1', 'fstype': 'ext4', 'options': 'rw,seclabel,relatime', 'size_total': 983429562368, 'size_available': 930320146432, 'block_size': 4096, 'block_total': 240095108, 'block_available': 227128942, 'block_used': 12966166, 'inode_total': 61054976, 'inode_available': 60985436, 'inode_used': 69540, 'uuid': 'bc802139-74bd-43cf-840d-e30da2bb57f8'}) => {"ansible_facts": {"root_disk": "/dev/sda"}, "ansible_loop_var": "item", "changed": false, "item": {"block_available": 227128942, "block_size": 4096, "block_total": 240095108, "block_used": 12966166, "device": "/dev/sda1", "fstype": "ext4", "inode_available": 60985436, "inode_total": 61054976, "inode_used": 69540, "mount": "/", "options": "rw,seclabel,relatime", "size_available": 930320146432, "size_total": 983429562368, "uuid": "bc802139-74bd-43cf-840d-e30da2bb57f8"}} 2021-11-11T20:44:56.849 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:56 +0000 (0:00:00.058) 0:07:17.105 ***** 2021-11-11T20:44:56.858 INFO:teuthology.task.ansible.out: TASK [testnode : Unmount any non-root mountpoints] ***************************** 2021-11-11T20:44:56.860 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'mount': '/', 'device': '/dev/sda1', 'fstype': 'ext4', 'options': 'rw,seclabel,relatime', 'size_total': 983429562368, 'size_available': 930320162816, 'block_size': 4096, 'block_total': 240095108, 'block_available': 227128946, 'block_used': 12966162, 'inode_total': 61054976, 'inode_available': 60985436, 'inode_used': 69540, 'uuid': 'bc802139-74bd-43cf-840d-e30da2bb57f8'}) => {"ansible_loop_var": "item", "changed": false, "item": {"block_available": 227128946, "block_size": 4096, "block_total": 240095108, "block_used": 12966162, "device": "/dev/sda1", "fstype": "ext4", "inode_available": 60985436, "inode_total": 61054976, "inode_used": 69540, "mount": "/", "options": "rw,seclabel,relatime", "size_available": 930320162816, "size_total": 983429562368, "uuid": "bc802139-74bd-43cf-840d-e30da2bb57f8"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:56.888 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:56 +0000 (0:00:00.038) 0:07:17.144 ***** 2021-11-11T20:44:56.915 INFO:teuthology.task.ansible.out: TASK [testnode : Wipe filesystems on non-root partitions] ********************** 2021-11-11T20:44:56.948 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:56 +0000 (0:00:00.059) 0:07:17.204 ***** 2021-11-11T20:44:57.040 INFO:teuthology.task.ansible.out: TASK [testnode : Compile list of non-root partitions] ************************** 2021-11-11T20:44:57.042 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": "lsblk --list --noheadings | grep part | grep -v sda | awk '{ print $1 }'", "delta": "0:00:00.005220", "end": "2021-11-11 20:44:57.020111", "rc": 0, "start": "2021-11-11 20:44:57.014891", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:44:57.064 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:57 +0000 (0:00:00.116) 0:07:17.320 ***** 2021-11-11T20:44:57.103 INFO:teuthology.task.ansible.out: TASK [testnode : Unmount any non-root mountpoints] ***************************** 2021-11-11T20:44:57.105 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'mount': '/', 'device': '/dev/sda1', 'fstype': 'ext4', 'options': 'rw,seclabel,relatime', 'size_total': 983429562368, 'size_available': 930320146432, 'block_size': 4096, 'block_total': 240095108, 'block_available': 227128942, 'block_used': 12966166, 'inode_total': 61054976, 'inode_available': 60985436, 'inode_used': 69540, 'uuid': 'bc802139-74bd-43cf-840d-e30da2bb57f8'}) => {"ansible_loop_var": "item", "changed": false, "item": {"block_available": 227128942, "block_size": 4096, "block_total": 240095108, "block_used": 12966166, "device": "/dev/sda1", "fstype": "ext4", "inode_available": 60985436, "inode_total": 61054976, "inode_used": 69540, "mount": "/", "options": "rw,seclabel,relatime", "size_available": 930320146432, "size_total": 983429562368, "uuid": "bc802139-74bd-43cf-840d-e30da2bb57f8"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:44:57.126 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:57 +0000 (0:00:00.062) 0:07:17.382 ***** 2021-11-11T20:44:57.153 INFO:teuthology.task.ansible.out: TASK [testnode : Wipe filesystems on non-root partitions] ********************** 2021-11-11T20:44:57.175 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:44:57 +0000 (0:00:00.048) 0:07:17.431 ***** 2021-11-11T20:44:58.305 INFO:teuthology.task.ansible.out: TASK [testnode : Zap all non-root disks] *************************************** 2021-11-11T20:44:58.307 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'dm-1', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_2', 'dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrl8Bw0N83ASULSve5G6krcQnf2ygY0pdoJ'], 'uuids': ['1f61c43d-2cb1-4798-a076-b039c5f2a286'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/dm-1 || sgdisk --zap-all /dev/dm-1", "delta": "0:00:01.136235", "end": "2021-11-11 20:44:58.271113", "item": {"key": "dm-1", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_2", "dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrl8Bw0N83ASULSve5G6krcQnf2ygY0pdoJ"], "labels": [], "masters": [], "uuids": ["1f61c43d-2cb1-4798-a076-b039c5f2a286"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:44:57.134878", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "Warning: The kernel is still using the old partition table.", "The new table will be used at the next reboot or after you", "run partprobe(8) or kpartx(8)", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:44:58.568 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'dm-1', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_2', 'dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyonRmpOZ1qV9KJ1dageQU7tWCgYnfoh8FM'], 'uuids': ['3c4596f7-6532-4fd9-9261-d784559e1061'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/dm-1 || sgdisk --zap-all /dev/dm-1", "delta": "0:00:01.167326", "end": "2021-11-11 20:44:58.531750", "item": {"key": "dm-1", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_2", "dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyonRmpOZ1qV9KJ1dageQU7tWCgYnfoh8FM"], "labels": [], "masters": [], "uuids": ["3c4596f7-6532-4fd9-9261-d784559e1061"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:44:57.364424", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "Warning: The kernel is still using the old partition table.", "The new table will be used at the next reboot or after you", "run partprobe(8) or kpartx(8)", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:44:59.551 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'nvme0n1', 'value': {'virtual': 1, 'links': {'ids': ['lvm-pv-uuid-uvGYi8-CGbG-hMNQ-Facr-q1ju-QIJ2-7xfP1Y', 'nvme-INTEL_SSDPEDMD400G4_PHFT620400NH400BGN', 'nvme-nvme.8086-504846543632303430304e4834303042474e-494e54454c205353445045444d443430304734-00000001'], 'uuids': [], 'labels': [], 'masters': ['dm-0', 'dm-1', 'dm-2', 'dm-3', 'dm-4']}, 'vendor': None, 'model': 'INTEL SSDPEDMD400G4', 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': 'none', 'sectors': '781422768', 'sectorsize': '512', 'size': '372.61 GB', 'host': 'Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)', 'holders': ['vg_nvme-lv_2', 'vg_nvme-lv_5', 'vg_nvme-lv_3', 'vg_nvme-lv_1', 'vg_nvme-lv_4']}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/nvme0n1 || sgdisk --zap-all /dev/nvme0n1", "delta": "0:00:01.054353", "end": "2021-11-11 20:44:59.516718", "item": {"key": "nvme0n1", "value": {"holders": ["vg_nvme-lv_2", "vg_nvme-lv_5", "vg_nvme-lv_3", "vg_nvme-lv_1", "vg_nvme-lv_4"], "host": "Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)", "links": {"ids": ["lvm-pv-uuid-uvGYi8-CGbG-hMNQ-Facr-q1ju-QIJ2-7xfP1Y", "nvme-INTEL_SSDPEDMD400G4_PHFT620400NH400BGN", "nvme-nvme.8086-504846543632303430304e4834303042474e-494e54454c205353445045444d443430304734-00000001"], "labels": [], "masters": ["dm-0", "dm-1", "dm-2", "dm-3", "dm-4"], "uuids": []}, "model": "INTEL SSDPEDMD400G4", "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "781422768", "sectorsize": "512", "size": "372.61 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:44:58.462365", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nGPT data structures destroyed! 2021-11-11T20:44:59.554 INFO:teuthology.task.ansible.out:You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:44:59.828 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'nvme0n1', 'value': {'virtual': 1, 'links': {'ids': ['lvm-pv-uuid-3JOdMY-i6gN-jzSd-xetu-E2C1-3RFb-OqLBsf', 'nvme-INTEL_SSDPEDMD400G4_PHFT620500P7400BGN', 'nvme-nvme.8086-50484654363230353030503734303042474e-494e54454c205353445045444d443430304734-00000001'], 'uuids': [], 'labels': [], 'masters': ['dm-0', 'dm-1', 'dm-2', 'dm-3', 'dm-4']}, 'vendor': None, 'model': 'INTEL SSDPEDMD400G4', 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': 'none', 'sectors': '781422768', 'sectorsize': '512', 'size': '372.61 GB', 'host': 'Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)', 'holders': ['vg_nvme-lv_2', 'vg_nvme-lv_5', 'vg_nvme-lv_3', 'vg_nvme-lv_1', 'vg_nvme-lv_4']}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/nvme0n1 || sgdisk --zap-all /dev/nvme0n1", "delta": "0:00:01.063293", "end": "2021-11-11 20:44:59.787928", "item": {"key": "nvme0n1", "value": {"holders": ["vg_nvme-lv_2", "vg_nvme-lv_5", "vg_nvme-lv_3", "vg_nvme-lv_1", "vg_nvme-lv_4"], "host": "Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)", "links": {"ids": ["lvm-pv-uuid-3JOdMY-i6gN-jzSd-xetu-E2C1-3RFb-OqLBsf", "nvme-INTEL_SSDPEDMD400G4_PHFT620500P7400BGN", "nvme-nvme.8086-50484654363230353030503734303042474e-494e54454c205353445045444d443430304734-00000001"], "labels": [], "masters": ["dm-0", "dm-1", "dm-2", "dm-3", "dm-4"], "uuids": []}, "model": "INTEL SSDPEDMD400G4", "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "781422768", "sectorsize": "512", "size": "372.61 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:44:58.724635", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nGPT data structures destroyed! 2021-11-11T20:44:59.830 INFO:teuthology.task.ansible.out:You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:45:00.783 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'dm-4', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_5', 'dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlF4MRkqQKYB28o80t8aEWOzQZxj8OMMqO'], 'uuids': ['246285f3-fb17-40da-b55d-1b02858eeba2'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '31252480', 'sectorsize': '512', 'size': '14.90 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/dm-4 || sgdisk --zap-all /dev/dm-4", "delta": "0:00:01.049499", "end": "2021-11-11 20:45:00.756314", "item": {"key": "dm-4", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_5", "dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlF4MRkqQKYB28o80t8aEWOzQZxj8OMMqO"], "labels": [], "masters": [], "uuids": ["246285f3-fb17-40da-b55d-1b02858eeba2"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "31252480", "sectorsize": "512", "size": "14.90 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:44:59.706815", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "Warning: The kernel is still using the old partition table.", "The new table will be used at the next reboot or after you", "run partprobe(8) or kpartx(8)", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:45:01.058 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'dm-4', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_5', 'dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyof2zpml909rw1GGl2i1f0RBjpWTEcYVC2'], 'uuids': ['6c07ff08-d63b-4bf9-bd5f-0e5399e56fd3'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '31252480', 'sectorsize': '512', 'size': '14.90 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/dm-4 || sgdisk --zap-all /dev/dm-4", "delta": "0:00:01.038827", "end": "2021-11-11 20:45:01.027269", "item": {"key": "dm-4", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_5", "dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyof2zpml909rw1GGl2i1f0RBjpWTEcYVC2"], "labels": [], "masters": [], "uuids": ["6c07ff08-d63b-4bf9-bd5f-0e5399e56fd3"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "31252480", "sectorsize": "512", "size": "14.90 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:44:59.988442", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "Warning: The kernel is still using the old partition table.", "The new table will be used at the next reboot or after you", "run partprobe(8) or kpartx(8)", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:45:02.033 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'dm-2', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_3', 'dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlfr0CjRGoLDcVNd1X8ma0JMuVwi1l4ivN'], 'uuids': ['481c3c33-51f5-4c37-b75b-18bf8d57c95c'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/dm-2 || sgdisk --zap-all /dev/dm-2", "delta": "0:00:01.046141", "end": "2021-11-11 20:45:01.995270", "item": {"key": "dm-2", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_3", "dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlfr0CjRGoLDcVNd1X8ma0JMuVwi1l4ivN"], "labels": [], "masters": [], "uuids": ["481c3c33-51f5-4c37-b75b-18bf8d57c95c"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:00.949129", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "Warning: The kernel is still using the old partition table.", "The new table will be used at the next reboot or after you", "run partprobe(8) or kpartx(8)", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:45:02.305 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'dm-2', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_3', 'dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyoIz2DJckase58pvEEHUV9iTZnMoYNnV8U'], 'uuids': ['8fd3d2f1-c3e2-46c1-93d2-93adb2d7617b'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/dm-2 || sgdisk --zap-all /dev/dm-2", "delta": "0:00:01.058152", "end": "2021-11-11 20:45:02.277863", "item": {"key": "dm-2", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_3", "dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyoIz2DJckase58pvEEHUV9iTZnMoYNnV8U"], "labels": [], "masters": [], "uuids": ["8fd3d2f1-c3e2-46c1-93d2-93adb2d7617b"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:01.219711", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "Warning: The kernel is still using the old partition table.", "The new table will be used at the next reboot or after you", "run partprobe(8) or kpartx(8)", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:45:03.270 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'dm-0', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_1', 'dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlB4qYaeCsfFFeBsb3moU3KjvKQNN4RUaC'], 'uuids': ['da8d5387-eada-497b-87ee-e3d39462eca1'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/dm-0 || sgdisk --zap-all /dev/dm-0", "delta": "0:00:01.056909", "end": "2021-11-11 20:45:03.238456", "item": {"key": "dm-0", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_1", "dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlB4qYaeCsfFFeBsb3moU3KjvKQNN4RUaC"], "labels": [], "masters": [], "uuids": ["da8d5387-eada-497b-87ee-e3d39462eca1"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:02.181547", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "Warning: The kernel is still using the old partition table.", "The new table will be used at the next reboot or after you", "run partprobe(8) or kpartx(8)", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:45:03.284 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'key': 'sda', 'value': {'virtual': 1, 'links': {'ids': ['ata-ST1000NM0033-9ZM173_Z1W5QGMP', 'scsi-0ATA_ST1000NM0033-9ZM_Z1W5QGMP', 'scsi-1ATA_ST1000NM0033-9ZM173_Z1W5QGMP', 'scsi-35000c50091dfbebc', 'scsi-SATA_ST1000NM0033-9ZM_Z1W5QGMP', 'wwn-0x5000c50091dfbebc'], 'uuids': [], 'labels': [], 'masters': []}, 'vendor': 'ATA', 'model': 'ST1000NM0033-9ZM', 'sas_address': None, 'sas_device_handle': None, 'serial': 'Z1W5QGMP', 'removable': '0', 'support_discard': '0', 'wwn': '0x5000c50091dfbebc', 'partitions': {'sda1': {'links': {'ids': ['ata-ST1000NM0033-9ZM173_Z1W5QGMP-part1', 'scsi-0ATA_ST1000NM0033-9ZM_Z1W5QGMP-part1', 'scsi-1ATA_ST1000NM0033-9ZM173_Z1W5QGMP-part1', 'scsi-35000c50091dfbebc-part1', 'scsi-SATA_ST1000NM0033-9ZM_Z1W5QGMP-part1', 'wwn-0x5000c50091dfbebc-part1'], 'uuids': ['bc802139-74bd-43cf-840d-e30da2bb57f8'], 'labels': [], 'masters': []}, 'start': '2048', 'sectors': '1953522688', 'sectorsize': 512, 'size': '931.51 GB', 'uuid': 'bc802139-74bd-43cf-840d-e30da2bb57f8', 'holders': []}}, 'rotational': '1', 'scheduler_mode': 'mq-deadline', 'sectors': '1953525168', 'sectorsize': '512', 'size': '931.51 GB', 'host': 'SATA controller: Intel Corporation C610/X99 series chipset 6-Port SATA Controller [AHCI mode] (rev 05)', 'holders': []}}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "sda", "value": {"holders": [], "host": "SATA controller: Intel Corporation C610/X99 series chipset 6-Port SATA Controller [AHCI mode] (rev 05)", "links": {"ids": ["ata-ST1000NM0033-9ZM173_Z1W5QGMP", "scsi-0ATA_ST1000NM0033-9ZM_Z1W5QGMP", "scsi-1ATA_ST1000NM0033-9ZM173_Z1W5QGMP", "scsi-35000c50091dfbebc", "scsi-SATA_ST1000NM0033-9ZM_Z1W5QGMP", "wwn-0x5000c50091dfbebc"], "labels": [], "masters": [], "uuids": []}, "model": "ST1000NM0033-9ZM", "partitions": {"sda1": {"holders": [], "links": {"ids": ["ata-ST1000NM0033-9ZM173_Z1W5QGMP-part1", "scsi-0ATA_ST1000NM0033-9ZM_Z1W5QGMP-part1", "scsi-1ATA_ST1000NM0033-9ZM173_ 2021-11-11T20:45:03.286 INFO:teuthology.task.ansible.out:Z1W5QGMP-part1", "scsi-35000c50091dfbebc-part1", "scsi-SATA_ST1000NM0033-9ZM_Z1W5QGMP-part1", "wwn-0x5000c50091dfbebc-part1"], "labels": [], "masters": [], "uuids": ["bc802139-74bd-43cf-840d-e30da2bb57f8"]}, "sectors": "1953522688", "sectorsize": 512, "size": "931.51 GB", "start": "2048", "uuid": "bc802139-74bd-43cf-840d-e30da2bb57f8"}}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "mq-deadline", "sectors": "1953525168", "sectorsize": "512", "serial": "Z1W5QGMP", "size": "931.51 GB", "support_discard": "0", "vendor": "ATA", "virtual": 1, "wwn": "0x5000c50091dfbebc"}}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:03.544 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'dm-0', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_1', 'dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyo2fo9oDLCXqa8L3DV2scoEZ9KohqHMJR3'], 'uuids': ['1e0929f0-9096-42af-895f-41f2ff522638'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/dm-0 || sgdisk --zap-all /dev/dm-0", "delta": "0:00:01.046523", "end": "2021-11-11 20:45:03.510741", "item": {"key": "dm-0", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_1", "dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyo2fo9oDLCXqa8L3DV2scoEZ9KohqHMJR3"], "labels": [], "masters": [], "uuids": ["1e0929f0-9096-42af-895f-41f2ff522638"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:02.464218", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "Warning: The kernel is still using the old partition table.", "The new table will be used at the next reboot or after you", "run partprobe(8) or kpartx(8)", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:45:03.546 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'key': 'sda', 'value': {'virtual': 1, 'links': {'ids': ['ata-ST1000NM0033-9ZM173_Z1W5XGNA', 'scsi-0ATA_ST1000NM0033-9ZM_Z1W5XGNA', 'scsi-1ATA_ST1000NM0033-9ZM173_Z1W5XGNA', 'scsi-35000c5009292894e', 'scsi-SATA_ST1000NM0033-9ZM_Z1W5XGNA', 'wwn-0x5000c5009292894e'], 'uuids': [], 'labels': [], 'masters': []}, 'vendor': 'ATA', 'model': 'ST1000NM0033-9ZM', 'sas_address': None, 'sas_device_handle': None, 'serial': 'Z1W5XGNA', 'removable': '0', 'support_discard': '0', 'wwn': '0x5000c5009292894e', 'partitions': {'sda1': {'links': {'ids': ['ata-ST1000NM0033-9ZM173_Z1W5XGNA-part1', 'scsi-0ATA_ST1000NM0033-9ZM_Z1W5XGNA-part1', 'scsi-1ATA_ST1000NM0033-9ZM173_Z1W5XGNA-part1', 'scsi-35000c5009292894e-part1', 'scsi-SATA_ST1000NM0033-9ZM_Z1W5XGNA-part1', 'wwn-0x5000c5009292894e-part1'], 'uuids': ['bc802139-74bd-43cf-840d-e30da2bb57f8'], 'labels': [], 'masters': []}, 'start': '2048', 'sectors': '1953522688', 'sectorsize': 512, 'size': '931.51 GB', 'uuid': 'bc802139-74bd-43cf-840d-e30da2bb57f8', 'holders': []}}, 'rotational': '1', 'scheduler_mode': 'mq-deadline', 'sectors': '1953525168', 'sectorsize': '512', 'size': '931.51 GB', 'host': 'SATA controller: Intel Corporation C610/X99 series chipset 6-Port SATA Controller [AHCI mode] (rev 05)', 'holders': []}}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "sda", "value": {"holders": [], "host": "SATA controller: Intel Corporation C610/X99 series chipset 6-Port SATA Controller [AHCI mode] (rev 05)", "links": {"ids": ["ata-ST1000NM0033-9ZM173_Z1W5XGNA", "scsi-0ATA_ST1000NM0033-9ZM_Z1W5XGNA", "scsi-1ATA_ST1000NM0033-9ZM173_Z1W5XGNA", "scsi-35000c5009292894e", "scsi-SATA_ST1000NM0033-9ZM_Z1W5XGNA", "wwn-0x5000c5009292894e"], "labels": [], "masters": [], "uuids": []}, "model": "ST1000NM0033-9ZM", "partitions": {"sda1": {"holders": [], "links": {"ids": ["ata-ST1000NM0033-9ZM173_Z1W5XGNA-part1", "scsi-0ATA_ST1000NM0033-9ZM_Z1W5XGNA-part1", "scsi-1ATA_ST1000NM0033-9ZM173_ 2021-11-11T20:45:03.548 INFO:teuthology.task.ansible.out:Z1W5XGNA-part1", "scsi-35000c5009292894e-part1", "scsi-SATA_ST1000NM0033-9ZM_Z1W5XGNA-part1", "wwn-0x5000c5009292894e-part1"], "labels": [], "masters": [], "uuids": ["bc802139-74bd-43cf-840d-e30da2bb57f8"]}, "sectors": "1953522688", "sectorsize": 512, "size": "931.51 GB", "start": "2048", "uuid": "bc802139-74bd-43cf-840d-e30da2bb57f8"}}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "mq-deadline", "sectors": "1953525168", "sectorsize": "512", "serial": "Z1W5XGNA", "size": "931.51 GB", "support_discard": "0", "vendor": "ATA", "virtual": 1, "wwn": "0x5000c5009292894e"}}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:04.513 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'dm-3', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_4', 'dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlS8sTroLgrbzWxb00bGYcWpKZ2LpOqXG0'], 'uuids': ['dd9bc926-bc35-4fe6-99dc-02d137ac4f44'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/dm-3 || sgdisk --zap-all /dev/dm-3", "delta": "0:00:01.056392", "end": "2021-11-11 20:45:04.487767", "item": {"key": "dm-3", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_4", "dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlS8sTroLgrbzWxb00bGYcWpKZ2LpOqXG0"], "labels": [], "masters": [], "uuids": ["dd9bc926-bc35-4fe6-99dc-02d137ac4f44"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:03.431375", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "Warning: The kernel is still using the old partition table.", "The new table will be used at the next reboot or after you", "run partprobe(8) or kpartx(8)", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:45:04.566 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:04 +0000 (0:00:07.391) 0:07:24.822 ***** 2021-11-11T20:45:04.786 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'dm-3', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_4', 'dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyoYOKngIQKwr8fbFHll2PgakJRGn7lonVe'], 'uuids': ['66b968ac-f58c-41c7-9cdb-8adbfea020c2'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "sgdisk --zap-all /dev/dm-3 || sgdisk --zap-all /dev/dm-3", "delta": "0:00:01.047759", "end": "2021-11-11 20:45:04.754320", "item": {"key": "dm-3", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_4", "dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyoYOKngIQKwr8fbFHll2PgakJRGn7lonVe"], "labels": [], "masters": [], "uuids": ["66b968ac-f58c-41c7-9cdb-8adbfea020c2"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:03.706561", "stderr": "", "stderr_lines": [], "stdout": "Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.", "stdout_lines": ["Creating new GPT entries.", "Warning: The kernel is still using the old partition table.", "The new table will be used at the next reboot or after you", "run partprobe(8) or kpartx(8)", "GPT data structures destroyed! You may now partition the disk using fdisk or", "other utilities."]} 2021-11-11T20:45:04.829 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:04 +0000 (0:00:00.263) 0:07:25.085 ***** 2021-11-11T20:45:04.970 INFO:teuthology.task.ansible.out: TASK [testnode : Blow away lingering OSD data and FSIDs] *********************** 2021-11-11T20:45:04.972 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'dm-1', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_2', 'dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrl8Bw0N83ASULSve5G6krcQnf2ygY0pdoJ'], 'uuids': ['1f61c43d-2cb1-4798-a076-b039c5f2a286'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/dm-1 bs=1M count=110", "delta": "0:00:00.144919", "end": "2021-11-11 20:45:04.940545", "item": {"key": "dm-1", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_2", "dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrl8Bw0N83ASULSve5G6krcQnf2ygY0pdoJ"], "labels": [], "masters": [], "uuids": ["1f61c43d-2cb1-4798-a076-b039c5f2a286"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:04.795626", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.142239 s, 811 MB/s", "stderr_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.142239 s, 811 MB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:05.204 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'nvme0n1', 'value': {'virtual': 1, 'links': {'ids': ['lvm-pv-uuid-uvGYi8-CGbG-hMNQ-Facr-q1ju-QIJ2-7xfP1Y', 'nvme-INTEL_SSDPEDMD400G4_PHFT620400NH400BGN', 'nvme-nvme.8086-504846543632303430304e4834303042474e-494e54454c205353445045444d443430304734-00000001'], 'uuids': [], 'labels': [], 'masters': ['dm-0', 'dm-1', 'dm-2', 'dm-3', 'dm-4']}, 'vendor': None, 'model': 'INTEL SSDPEDMD400G4', 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': 'none', 'sectors': '781422768', 'sectorsize': '512', 'size': '372.61 GB', 'host': 'Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)', 'holders': ['vg_nvme-lv_2', 'vg_nvme-lv_5', 'vg_nvme-lv_3', 'vg_nvme-lv_1', 'vg_nvme-lv_4']}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/nvme0n1 bs=1M count=110", "delta": "0:00:00.040600", "end": "2021-11-11 20:45:05.176993", "item": {"key": "nvme0n1", "value": {"holders": ["vg_nvme-lv_2", "vg_nvme-lv_5", "vg_nvme-lv_3", "vg_nvme-lv_1", "vg_nvme-lv_4"], "host": "Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)", "links": {"ids": ["lvm-pv-uuid-uvGYi8-CGbG-hMNQ-Facr-q1ju-QIJ2-7xfP1Y", "nvme-INTEL_SSDPEDMD400G4_PHFT620400NH400BGN", "nvme-nvme.8086-504846543632303430304e4834303042474e-494e54454c205353445045444d443430304734-00000001"], "labels": [], "masters": ["dm-0", "dm-1", "dm-2", "dm-3", "dm-4"], "uuids": []}, "model": "INTEL SSDPEDMD400G4", "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "781422768", "sectorsize": "512", "size": "372.61 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:05.136393", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.0378542 s, 3.0 GB/s", "std 2021-11-11T20:45:05.206 INFO:teuthology.task.ansible.out:err_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.0378542 s, 3.0 GB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:05.216 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'dm-1', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_2', 'dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyonRmpOZ1qV9KJ1dageQU7tWCgYnfoh8FM'], 'uuids': ['3c4596f7-6532-4fd9-9261-d784559e1061'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/dm-1 bs=1M count=110", "delta": "0:00:00.154279", "end": "2021-11-11 20:45:05.188219", "item": {"key": "dm-1", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_2", "dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyonRmpOZ1qV9KJ1dageQU7tWCgYnfoh8FM"], "labels": [], "masters": [], "uuids": ["3c4596f7-6532-4fd9-9261-d784559e1061"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:05.033940", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.151415 s, 762 MB/s", "stderr_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.151415 s, 762 MB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:05.439 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'nvme0n1', 'value': {'virtual': 1, 'links': {'ids': ['lvm-pv-uuid-3JOdMY-i6gN-jzSd-xetu-E2C1-3RFb-OqLBsf', 'nvme-INTEL_SSDPEDMD400G4_PHFT620500P7400BGN', 'nvme-nvme.8086-50484654363230353030503734303042474e-494e54454c205353445045444d443430304734-00000001'], 'uuids': [], 'labels': [], 'masters': ['dm-0', 'dm-1', 'dm-2', 'dm-3', 'dm-4']}, 'vendor': None, 'model': 'INTEL SSDPEDMD400G4', 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': 'none', 'sectors': '781422768', 'sectorsize': '512', 'size': '372.61 GB', 'host': 'Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)', 'holders': ['vg_nvme-lv_2', 'vg_nvme-lv_5', 'vg_nvme-lv_3', 'vg_nvme-lv_1', 'vg_nvme-lv_4']}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/nvme0n1 bs=1M count=110", "delta": "0:00:00.039631", "end": "2021-11-11 20:45:05.409599", "item": {"key": "nvme0n1", "value": {"holders": ["vg_nvme-lv_2", "vg_nvme-lv_5", "vg_nvme-lv_3", "vg_nvme-lv_1", "vg_nvme-lv_4"], "host": "Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)", "links": {"ids": ["lvm-pv-uuid-3JOdMY-i6gN-jzSd-xetu-E2C1-3RFb-OqLBsf", "nvme-INTEL_SSDPEDMD400G4_PHFT620500P7400BGN", "nvme-nvme.8086-50484654363230353030503734303042474e-494e54454c205353445045444d443430304734-00000001"], "labels": [], "masters": ["dm-0", "dm-1", "dm-2", "dm-3", "dm-4"], "uuids": []}, "model": "INTEL SSDPEDMD400G4", "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "none", "sectors": "781422768", "sectorsize": "512", "size": "372.61 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:05.369968", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.0368995 s, 3.1 GB/s", "std 2021-11-11T20:45:05.442 INFO:teuthology.task.ansible.out:err_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.0368995 s, 3.1 GB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:05.535 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'dm-4', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_5', 'dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlF4MRkqQKYB28o80t8aEWOzQZxj8OMMqO'], 'uuids': ['246285f3-fb17-40da-b55d-1b02858eeba2'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '31252480', 'sectorsize': '512', 'size': '14.90 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/dm-4 bs=1M count=110", "delta": "0:00:00.146602", "end": "2021-11-11 20:45:05.504312", "item": {"key": "dm-4", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_5", "dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlF4MRkqQKYB28o80t8aEWOzQZxj8OMMqO"], "labels": [], "masters": [], "uuids": ["246285f3-fb17-40da-b55d-1b02858eeba2"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "31252480", "sectorsize": "512", "size": "14.90 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:05.357710", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.14383 s, 802 MB/s", "stderr_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.14383 s, 802 MB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:05.764 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'dm-4', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_5', 'dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyof2zpml909rw1GGl2i1f0RBjpWTEcYVC2'], 'uuids': ['6c07ff08-d63b-4bf9-bd5f-0e5399e56fd3'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '31252480', 'sectorsize': '512', 'size': '14.90 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/dm-4 bs=1M count=110", "delta": "0:00:00.148545", "end": "2021-11-11 20:45:05.738294", "item": {"key": "dm-4", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_5", "dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyof2zpml909rw1GGl2i1f0RBjpWTEcYVC2"], "labels": [], "masters": [], "uuids": ["6c07ff08-d63b-4bf9-bd5f-0e5399e56fd3"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "31252480", "sectorsize": "512", "size": "14.90 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:05.589749", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.145855 s, 791 MB/s", "stderr_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.145855 s, 791 MB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:05.850 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'dm-2', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_3', 'dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlfr0CjRGoLDcVNd1X8ma0JMuVwi1l4ivN'], 'uuids': ['481c3c33-51f5-4c37-b75b-18bf8d57c95c'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/dm-2 bs=1M count=110", "delta": "0:00:00.143581", "end": "2021-11-11 20:45:05.823317", "item": {"key": "dm-2", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_3", "dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlfr0CjRGoLDcVNd1X8ma0JMuVwi1l4ivN"], "labels": [], "masters": [], "uuids": ["481c3c33-51f5-4c37-b75b-18bf8d57c95c"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:05.679736", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.140983 s, 818 MB/s", "stderr_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.140983 s, 818 MB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:06.087 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'dm-2', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_3', 'dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyoIz2DJckase58pvEEHUV9iTZnMoYNnV8U'], 'uuids': ['8fd3d2f1-c3e2-46c1-93d2-93adb2d7617b'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/dm-2 bs=1M count=110", "delta": "0:00:00.147039", "end": "2021-11-11 20:45:06.056814", "item": {"key": "dm-2", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_3", "dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyoIz2DJckase58pvEEHUV9iTZnMoYNnV8U"], "labels": [], "masters": [], "uuids": ["8fd3d2f1-c3e2-46c1-93d2-93adb2d7617b"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:05.909775", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.144379 s, 799 MB/s", "stderr_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.144379 s, 799 MB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:06.194 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'dm-0', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_1', 'dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlB4qYaeCsfFFeBsb3moU3KjvKQNN4RUaC'], 'uuids': ['da8d5387-eada-497b-87ee-e3d39462eca1'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/dm-0 bs=1M count=110", "delta": "0:00:00.157595", "end": "2021-11-11 20:45:06.164296", "item": {"key": "dm-0", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_1", "dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlB4qYaeCsfFFeBsb3moU3KjvKQNN4RUaC"], "labels": [], "masters": [], "uuids": ["da8d5387-eada-497b-87ee-e3d39462eca1"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:06.006701", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.154932 s, 744 MB/s", "stderr_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.154932 s, 744 MB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:06.195 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'key': 'sda', 'value': {'virtual': 1, 'links': {'ids': ['ata-ST1000NM0033-9ZM173_Z1W5QGMP', 'scsi-0ATA_ST1000NM0033-9ZM_Z1W5QGMP', 'scsi-1ATA_ST1000NM0033-9ZM173_Z1W5QGMP', 'scsi-35000c50091dfbebc', 'scsi-SATA_ST1000NM0033-9ZM_Z1W5QGMP', 'wwn-0x5000c50091dfbebc'], 'uuids': [], 'labels': [], 'masters': []}, 'vendor': 'ATA', 'model': 'ST1000NM0033-9ZM', 'sas_address': None, 'sas_device_handle': None, 'serial': 'Z1W5QGMP', 'removable': '0', 'support_discard': '0', 'wwn': '0x5000c50091dfbebc', 'partitions': {'sda1': {'links': {'ids': ['ata-ST1000NM0033-9ZM173_Z1W5QGMP-part1', 'scsi-0ATA_ST1000NM0033-9ZM_Z1W5QGMP-part1', 'scsi-1ATA_ST1000NM0033-9ZM173_Z1W5QGMP-part1', 'scsi-35000c50091dfbebc-part1', 'scsi-SATA_ST1000NM0033-9ZM_Z1W5QGMP-part1', 'wwn-0x5000c50091dfbebc-part1'], 'uuids': ['bc802139-74bd-43cf-840d-e30da2bb57f8'], 'labels': [], 'masters': []}, 'start': '2048', 'sectors': '1953522688', 'sectorsize': 512, 'size': '931.51 GB', 'uuid': 'bc802139-74bd-43cf-840d-e30da2bb57f8', 'holders': []}}, 'rotational': '1', 'scheduler_mode': 'mq-deadline', 'sectors': '1953525168', 'sectorsize': '512', 'size': '931.51 GB', 'host': 'SATA controller: Intel Corporation C610/X99 series chipset 6-Port SATA Controller [AHCI mode] (rev 05)', 'holders': []}}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "sda", "value": {"holders": [], "host": "SATA controller: Intel Corporation C610/X99 series chipset 6-Port SATA Controller [AHCI mode] (rev 05)", "links": {"ids": ["ata-ST1000NM0033-9ZM173_Z1W5QGMP", "scsi-0ATA_ST1000NM0033-9ZM_Z1W5QGMP", "scsi-1ATA_ST1000NM0033-9ZM173_Z1W5QGMP", "scsi-35000c50091dfbebc", "scsi-SATA_ST1000NM0033-9ZM_Z1W5QGMP", "wwn-0x5000c50091dfbebc"], "labels": [], "masters": [], "uuids": []}, "model": "ST1000NM0033-9ZM", "partitions": {"sda1": {"holders": [], "links": {"ids": ["ata-ST1000NM0033-9ZM173_Z1W5QGMP-part1", "scsi-0ATA_ST1000NM0033-9ZM_Z1W5QGMP-part1", "scsi-1ATA_ST1000NM0033-9ZM173_ 2021-11-11T20:45:06.197 INFO:teuthology.task.ansible.out:Z1W5QGMP-part1", "scsi-35000c50091dfbebc-part1", "scsi-SATA_ST1000NM0033-9ZM_Z1W5QGMP-part1", "wwn-0x5000c50091dfbebc-part1"], "labels": [], "masters": [], "uuids": ["bc802139-74bd-43cf-840d-e30da2bb57f8"]}, "sectors": "1953522688", "sectorsize": 512, "size": "931.51 GB", "start": "2048", "uuid": "bc802139-74bd-43cf-840d-e30da2bb57f8"}}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "mq-deadline", "sectors": "1953525168", "sectorsize": "512", "serial": "Z1W5QGMP", "size": "931.51 GB", "support_discard": "0", "vendor": "ATA", "virtual": 1, "wwn": "0x5000c50091dfbebc"}}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:06.420 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'dm-0', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_1', 'dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyo2fo9oDLCXqa8L3DV2scoEZ9KohqHMJR3'], 'uuids': ['1e0929f0-9096-42af-895f-41f2ff522638'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/dm-0 bs=1M count=110", "delta": "0:00:00.147533", "end": "2021-11-11 20:45:06.393227", "item": {"key": "dm-0", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_1", "dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyo2fo9oDLCXqa8L3DV2scoEZ9KohqHMJR3"], "labels": [], "masters": [], "uuids": ["1e0929f0-9096-42af-895f-41f2ff522638"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:06.245694", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.144816 s, 796 MB/s", "stderr_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.144816 s, 796 MB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:06.422 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'key': 'sda', 'value': {'virtual': 1, 'links': {'ids': ['ata-ST1000NM0033-9ZM173_Z1W5XGNA', 'scsi-0ATA_ST1000NM0033-9ZM_Z1W5XGNA', 'scsi-1ATA_ST1000NM0033-9ZM173_Z1W5XGNA', 'scsi-35000c5009292894e', 'scsi-SATA_ST1000NM0033-9ZM_Z1W5XGNA', 'wwn-0x5000c5009292894e'], 'uuids': [], 'labels': [], 'masters': []}, 'vendor': 'ATA', 'model': 'ST1000NM0033-9ZM', 'sas_address': None, 'sas_device_handle': None, 'serial': 'Z1W5XGNA', 'removable': '0', 'support_discard': '0', 'wwn': '0x5000c5009292894e', 'partitions': {'sda1': {'links': {'ids': ['ata-ST1000NM0033-9ZM173_Z1W5XGNA-part1', 'scsi-0ATA_ST1000NM0033-9ZM_Z1W5XGNA-part1', 'scsi-1ATA_ST1000NM0033-9ZM173_Z1W5XGNA-part1', 'scsi-35000c5009292894e-part1', 'scsi-SATA_ST1000NM0033-9ZM_Z1W5XGNA-part1', 'wwn-0x5000c5009292894e-part1'], 'uuids': ['bc802139-74bd-43cf-840d-e30da2bb57f8'], 'labels': [], 'masters': []}, 'start': '2048', 'sectors': '1953522688', 'sectorsize': 512, 'size': '931.51 GB', 'uuid': 'bc802139-74bd-43cf-840d-e30da2bb57f8', 'holders': []}}, 'rotational': '1', 'scheduler_mode': 'mq-deadline', 'sectors': '1953525168', 'sectorsize': '512', 'size': '931.51 GB', 'host': 'SATA controller: Intel Corporation C610/X99 series chipset 6-Port SATA Controller [AHCI mode] (rev 05)', 'holders': []}}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "sda", "value": {"holders": [], "host": "SATA controller: Intel Corporation C610/X99 series chipset 6-Port SATA Controller [AHCI mode] (rev 05)", "links": {"ids": ["ata-ST1000NM0033-9ZM173_Z1W5XGNA", "scsi-0ATA_ST1000NM0033-9ZM_Z1W5XGNA", "scsi-1ATA_ST1000NM0033-9ZM173_Z1W5XGNA", "scsi-35000c5009292894e", "scsi-SATA_ST1000NM0033-9ZM_Z1W5XGNA", "wwn-0x5000c5009292894e"], "labels": [], "masters": [], "uuids": []}, "model": "ST1000NM0033-9ZM", "partitions": {"sda1": {"holders": [], "links": {"ids": ["ata-ST1000NM0033-9ZM173_Z1W5XGNA-part1", "scsi-0ATA_ST1000NM0033-9ZM_Z1W5XGNA-part1", "scsi-1ATA_ST1000NM0033-9ZM173_ 2021-11-11T20:45:06.424 INFO:teuthology.task.ansible.out:Z1W5XGNA-part1", "scsi-35000c5009292894e-part1", "scsi-SATA_ST1000NM0033-9ZM_Z1W5XGNA-part1", "wwn-0x5000c5009292894e-part1"], "labels": [], "masters": [], "uuids": ["bc802139-74bd-43cf-840d-e30da2bb57f8"]}, "sectors": "1953522688", "sectorsize": 512, "size": "931.51 GB", "start": "2048", "uuid": "bc802139-74bd-43cf-840d-e30da2bb57f8"}}, "removable": "0", "rotational": "1", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "mq-deadline", "sectors": "1953525168", "sectorsize": "512", "serial": "Z1W5XGNA", "size": "931.51 GB", "support_discard": "0", "vendor": "ATA", "virtual": 1, "wwn": "0x5000c5009292894e"}}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:06.522 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'dm-3', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_4', 'dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlS8sTroLgrbzWxb00bGYcWpKZ2LpOqXG0'], 'uuids': ['dd9bc926-bc35-4fe6-99dc-02d137ac4f44'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/dm-3 bs=1M count=110", "delta": "0:00:00.152852", "end": "2021-11-11 20:45:06.495284", "item": {"key": "dm-3", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_4", "dm-uuid-LVM-4yoD8qLE9C5QE6rjQ2Q25gwcXhkyIsrlS8sTroLgrbzWxb00bGYcWpKZ2LpOqXG0"], "labels": [], "masters": [], "uuids": ["dd9bc926-bc35-4fe6-99dc-02d137ac4f44"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:06.342432", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.150226 s, 768 MB/s", "stderr_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.150226 s, 768 MB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:06.551 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:06 +0000 (0:00:01.721) 0:07:26.807 ***** 2021-11-11T20:45:06.748 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'dm-3', 'value': {'virtual': 1, 'links': {'ids': ['dm-name-vg_nvme-lv_4', 'dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyoYOKngIQKwr8fbFHll2PgakJRGn7lonVe'], 'uuids': ['66b968ac-f58c-41c7-9cdb-8adbfea020c2'], 'labels': [], 'masters': []}, 'vendor': None, 'model': None, 'sas_address': None, 'sas_device_handle': None, 'removable': '0', 'support_discard': '512', 'partitions': {}, 'rotational': '0', 'scheduler_mode': '', 'sectors': '187490304', 'sectorsize': '512', 'size': '89.40 GB', 'host': '', 'holders': []}}) => {"ansible_loop_var": "item", "changed": true, "cmd": "dd if=/dev/zero of=/dev/dm-3 bs=1M count=110", "delta": "0:00:00.149846", "end": "2021-11-11 20:45:06.725093", "item": {"key": "dm-3", "value": {"holders": [], "host": "", "links": {"ids": ["dm-name-vg_nvme-lv_4", "dm-uuid-LVM-CxFqcSqwtl9cDdONhBY9MY0VNSkdZGyoYOKngIQKwr8fbFHll2PgakJRGn7lonVe"], "labels": [], "masters": [], "uuids": ["66b968ac-f58c-41c7-9cdb-8adbfea020c2"]}, "model": null, "partitions": {}, "removable": "0", "rotational": "0", "sas_address": null, "sas_device_handle": null, "scheduler_mode": "", "sectors": "187490304", "sectorsize": "512", "size": "89.40 GB", "support_discard": "512", "vendor": null, "virtual": 1}}, "rc": 0, "start": "2021-11-11 20:45:06.575247", "stderr": "110+0 records in\n110+0 records out\n115343360 bytes (115 MB, 110 MiB) copied, 0.147161 s, 784 MB/s", "stderr_lines": ["110+0 records in", "110+0 records out", "115343360 bytes (115 MB, 110 MiB) copied, 0.147161 s, 784 MB/s"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:06.781 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:06 +0000 (0:00:00.229) 0:07:27.036 ***** 2021-11-11T20:45:06.812 INFO:teuthology.task.ansible.out: TASK [testnode : Remove all LVM data] ****************************************** changed: [smithi096.front.sepia.ceph.com] => {"attempts": 1, "changed": true, "cmd": "dmsetup remove_all --force", "delta": "0:00:00.060803", "end": "2021-11-11 20:45:06.786278", "rc": 0, "start": "2021-11-11 20:45:06.725475", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:06.838 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:06 +0000 (0:00:00.056) 0:07:27.093 ***** 2021-11-11T20:45:07.026 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"attempts": 1, "changed": true, "cmd": "dmsetup remove_all --force", "delta": "0:00:00.051412", "end": "2021-11-11 20:45:07.000589", "rc": 0, "start": "2021-11-11 20:45:06.949177", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:07.050 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.212) 0:07:27.306 ***** 2021-11-11T20:45:07.068 INFO:teuthology.task.ansible.out: TASK [testnode : Check for physical volumes] *********************************** 2021-11-11T20:45:07.070 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": "pvdisplay | grep 'PV Name' | awk '{ print $3 }'", "delta": "0:00:00.040931", "end": "2021-11-11 20:45:07.043234", "rc": 0, "start": "2021-11-11 20:45:07.002303", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:07.093 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.042) 0:07:27.349 ***** 2021-11-11T20:45:07.122 INFO:teuthology.task.ansible.out: TASK [testnode : Remove physical volumes] ************************************** 2021-11-11T20:45:07.150 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.056) 0:07:27.405 ***** 2021-11-11T20:45:07.179 INFO:teuthology.task.ansible.out: TASK [testnode : Write a new partition table to {{ item.value.device }}] ******* skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.210 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.060) 0:07:27.465 ***** 2021-11-11T20:45:07.272 INFO:teuthology.task.ansible.out: TASK [testnode : Write partition entries to {{ item.0.device }}] *************** 2021-11-11T20:45:07.274 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.297 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.087) 0:07:27.553 ***** 2021-11-11T20:45:07.307 INFO:teuthology.task.ansible.out: TASK [testnode : Check for physical volumes] *********************************** 2021-11-11T20:45:07.309 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": "pvdisplay | grep 'PV Name' | awk '{ print $3 }'", "delta": "0:00:00.043768", "end": "2021-11-11 20:45:07.266691", "rc": 0, "start": "2021-11-11 20:45:07.222923", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:07.332 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.035) 0:07:27.588 ***** 2021-11-11T20:45:07.339 INFO:teuthology.task.ansible.out: TASK [testnode : Erase /scratch_devs so we know it's accurate] ***************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.366 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.033) 0:07:27.622 ***** 2021-11-11T20:45:07.373 INFO:teuthology.task.ansible.out: TASK [testnode : Remove physical volumes] ************************************** 2021-11-11T20:45:07.399 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.033) 0:07:27.655 ***** 2021-11-11T20:45:07.411 INFO:teuthology.task.ansible.out: TASK [testnode : Write /scratch_devs for {{ item.0.device }}] ****************** 2021-11-11T20:45:07.413 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.444 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.045) 0:07:27.700 ***** 2021-11-11T20:45:07.454 INFO:teuthology.task.ansible.out: TASK [testnode : Write a new partition table to {{ item.value.device }}] ******* 2021-11-11T20:45:07.456 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.498 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.051) 0:07:27.751 ***** 2021-11-11T20:45:07.535 INFO:teuthology.task.ansible.out: TASK [testnode : Write partition entries to {{ item.0.device }}] *************** 2021-11-11T20:45:07.537 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.560 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.064) 0:07:27.816 ***** 2021-11-11T20:45:07.591 INFO:teuthology.task.ansible.out: TASK [testnode : Erase /scratch_devs so we know it's accurate] ***************** 2021-11-11T20:45:07.592 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.617 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.056) 0:07:27.873 ***** 2021-11-11T20:45:07.644 INFO:teuthology.task.ansible.out: TASK [testnode : Edit lvm.conf to support LVM on kRBD.] ************************ 2021-11-11T20:45:07.646 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"backup": "", "changed": false, "msg": ""} TASK [testnode : Write /scratch_devs for {{ item.0.device }}] ****************** skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.668 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.051) 0:07:27.924 ***** 2021-11-11T20:45:07.689 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.020) 0:07:27.945 ***** 2021-11-11T20:45:07.707 INFO:teuthology.task.ansible.out: TASK [testnode : Set root disk] ************************************************ 2021-11-11T20:45:07.709 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'mount': '/', 'device': '/dev/sda1', 'fstype': 'ext4', 'options': 'rw,seclabel,relatime', 'size_total': 983429562368, 'size_available': 930320162816, 'block_size': 4096, 'block_total': 240095108, 'block_available': 227128946, 'block_used': 12966162, 'inode_total': 61054976, 'inode_available': 60985436, 'inode_used': 69540, 'uuid': 'bc802139-74bd-43cf-840d-e30da2bb57f8'}) => {"ansible_loop_var": "item", "changed": false, "item": {"block_available": 227128946, "block_size": 4096, "block_total": 240095108, "block_used": 12966162, "device": "/dev/sda1", "fstype": "ext4", "inode_available": 60985436, "inode_total": 61054976, "inode_used": 69540, "mount": "/", "options": "rw,seclabel,relatime", "size_available": 930320162816, "size_total": 983429562368, "uuid": "bc802139-74bd-43cf-840d-e30da2bb57f8"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.734 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.045) 0:07:27.990 ***** 2021-11-11T20:45:07.776 INFO:teuthology.task.ansible.out: TASK [testnode : Combine list of non-root disks] ******************************* skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.797 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.063) 0:07:28.053 ***** 2021-11-11T20:45:07.835 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** 2021-11-11T20:45:07.837 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.858 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.059) 0:07:28.113 ***** 2021-11-11T20:45:07.885 INFO:teuthology.task.ansible.out: TASK [testnode : Edit lvm.conf to support LVM on kRBD.] ************************ 2021-11-11T20:45:07.887 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"backup": "", "changed": false, "msg": ""} TASK [testnode : set_fact] ***************************************************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.908 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.050) 0:07:28.163 ***** 2021-11-11T20:45:07.929 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.019) 0:07:28.183 ***** 2021-11-11T20:45:07.945 INFO:teuthology.task.ansible.out: TASK [testnode : Create volume_groups dict] ************************************ 2021-11-11T20:45:07.947 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:07.969 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:07 +0000 (0:00:00.041) 0:07:28.225 ***** 2021-11-11T20:45:07.977 INFO:teuthology.task.ansible.out: TASK [testnode : Set root disk] ************************************************ 2021-11-11T20:45:07.979 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'mount': '/', 'device': '/dev/sda1', 'fstype': 'ext4', 'options': 'rw,seclabel,relatime', 'size_total': 983429562368, 'size_available': 930320146432, 'block_size': 4096, 'block_total': 240095108, 'block_available': 227128942, 'block_used': 12966166, 'inode_total': 61054976, 'inode_available': 60985436, 'inode_used': 69540, 'uuid': 'bc802139-74bd-43cf-840d-e30da2bb57f8'}) => {"ansible_loop_var": "item", "changed": false, "item": {"block_available": 227128942, "block_size": 4096, "block_total": 240095108, "block_used": 12966166, "device": "/dev/sda1", "fstype": "ext4", "inode_available": 60985436, "inode_total": 61054976, "inode_used": 69540, "mount": "/", "options": "rw,seclabel,relatime", "size_available": 930320146432, "size_total": 983429562368, "uuid": "bc802139-74bd-43cf-840d-e30da2bb57f8"}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:08.012 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:08 +0000 (0:00:00.042) 0:07:28.268 ***** 2021-11-11T20:45:08.019 INFO:teuthology.task.ansible.out: TASK [testnode : Determine desired logical volume percentage size] ************* 2021-11-11T20:45:08.021 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:08.045 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:08 +0000 (0:00:00.032) 0:07:28.300 ***** 2021-11-11T20:45:08.053 INFO:teuthology.task.ansible.out: TASK [testnode : Combine list of non-root disks] ******************************* 2021-11-11T20:45:08.054 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:08.077 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:08 +0000 (0:00:00.032) 0:07:28.333 ***** 2021-11-11T20:45:08.084 INFO:teuthology.task.ansible.out: TASK [testnode : Create logical_volumes dict] ********************************** 2021-11-11T20:45:08.085 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:08.110 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:08 +0000 (0:00:00.032) 0:07:28.365 ***** 2021-11-11T20:45:08.117 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** 2021-11-11T20:45:08.118 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:08.144 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:08 +0000 (0:00:00.034) 0:07:28.400 ***** 2021-11-11T20:45:08.183 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** 2021-11-11T20:45:08.185 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:08.210 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:08 +0000 (0:00:00.065) 0:07:28.465 ***** 2021-11-11T20:45:08.237 INFO:teuthology.task.ansible.out: TASK [testnode : Create volume_groups dict] ************************************ 2021-11-11T20:45:08.239 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:08.260 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:08 +0000 (0:00:00.050) 0:07:28.516 ***** 2021-11-11T20:45:08.298 INFO:teuthology.task.ansible.out: TASK [testnode : Determine desired logical volume percentage size] ************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:08.326 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:08 +0000 (0:00:00.066) 0:07:28.582 ***** 2021-11-11T20:45:08.354 INFO:teuthology.task.ansible.out: TASK [testnode : Create logical_volumes dict] ********************************** 2021-11-11T20:45:08.356 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:08.377 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:08 +0000 (0:00:00.050) 0:07:28.633 ***** 2021-11-11T20:45:08.615 INFO:teuthology.task.ansible.out: TASK [testnode : Create volume group(s)] *************************************** changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'vg_nvme', 'value': {'pvs': '/dev/nvme0n1'}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "vg_nvme", "value": {"pvs": "/dev/nvme0n1"}}} 2021-11-11T20:45:08.641 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:08 +0000 (0:00:00.264) 0:07:28.897 ***** 2021-11-11T20:45:08.802 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'vg_nvme', 'value': {'pvs': '/dev/nvme0n1'}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "vg_nvme", "value": {"pvs": "/dev/nvme0n1"}}} 2021-11-11T20:45:08.829 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:08 +0000 (0:00:00.188) 0:07:29.085 ***** 2021-11-11T20:45:09.112 INFO:teuthology.task.ansible.out: TASK [testnode : Create logical volume(s)] ************************************* 2021-11-11T20:45:09.114 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'lv_1', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "lv_1", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": ""} 2021-11-11T20:45:09.220 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'lv_1', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "lv_1", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": ""} 2021-11-11T20:45:09.462 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'lv_2', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "lv_2", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": ""} 2021-11-11T20:45:09.589 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'lv_2', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "lv_2", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": ""} 2021-11-11T20:45:09.788 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'lv_3', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "lv_3", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": ""} 2021-11-11T20:45:09.936 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'lv_3', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "lv_3", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": ""} 2021-11-11T20:45:10.136 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'lv_4', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "lv_4", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": ""} 2021-11-11T20:45:10.263 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'lv_4', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "lv_4", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": ""} 2021-11-11T20:45:10.498 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'lv_5', 'value': {'vg': 'vg_nvme', 'size': '14.9G'}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "lv_5", "value": {"size": "14.9G", "vg": "vg_nvme"}}, "msg": ""} 2021-11-11T20:45:10.528 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:10 +0000 (0:00:01.698) 0:07:30.784 ***** 2021-11-11T20:45:10.620 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'lv_5', 'value': {'vg': 'vg_nvme', 'size': '14.9G'}}) => {"ansible_loop_var": "item", "changed": true, "item": {"key": "lv_5", "value": {"size": "14.9G", "vg": "vg_nvme"}}, "msg": ""} 2021-11-11T20:45:10.649 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:10 +0000 (0:00:00.121) 0:07:30.905 ***** 2021-11-11T20:45:10.720 INFO:teuthology.task.ansible.out: TASK [testnode : Erase /scratch_devs so we know it's accurate] ***************** 2021-11-11T20:45:10.722 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "path": "/scratch_devs", "state": "absent"} 2021-11-11T20:45:10.745 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:10 +0000 (0:00:00.095) 0:07:31.001 ***** 2021-11-11T20:45:10.847 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "path": "/scratch_devs", "state": "absent"} 2021-11-11T20:45:10.872 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:10 +0000 (0:00:00.127) 0:07:31.128 ***** 2021-11-11T20:45:10.964 INFO:teuthology.task.ansible.out: TASK [testnode : Write /scratch_devs] ****************************************** 2021-11-11T20:45:10.966 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'lv_1', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "backup": "", "changed": true, "item": {"key": "lv_1", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": "line added"} 2021-11-11T20:45:11.092 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'lv_1', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "backup": "", "changed": true, "item": {"key": "lv_1", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": "line added"} 2021-11-11T20:45:11.136 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'lv_2', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "backup": "", "changed": true, "item": {"key": "lv_2", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": "line added"} 2021-11-11T20:45:11.274 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'lv_2', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "backup": "", "changed": true, "item": {"key": "lv_2", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": "line added"} 2021-11-11T20:45:11.308 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'lv_3', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "backup": "", "changed": true, "item": {"key": "lv_3", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": "line added"} 2021-11-11T20:45:11.460 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'lv_3', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "backup": "", "changed": true, "item": {"key": "lv_3", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": "line added"} 2021-11-11T20:45:11.482 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => (item={'key': 'lv_4', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "backup": "", "changed": true, "item": {"key": "lv_4", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": "line added"} 2021-11-11T20:45:11.493 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'key': 'lv_5', 'value': {'vg': 'vg_nvme', 'size': '14.9G'}}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "lv_5", "value": {"size": "14.9G", "vg": "vg_nvme"}}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:11.521 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:11 +0000 (0:00:00.649) 0:07:31.777 ***** 2021-11-11T20:45:11.561 INFO:teuthology.task.ansible.out: TASK [testnode : Create filesystems] ******************************************* 2021-11-11T20:45:11.563 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:11.586 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:11 +0000 (0:00:00.064) 0:07:31.841 ***** 2021-11-11T20:45:11.624 INFO:teuthology.task.ansible.out: TASK [testnode : Mount filesystems] ******************************************** 2021-11-11T20:45:11.626 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:11.649 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:11 +0000 (0:00:00.063) 0:07:31.905 ***** 2021-11-11T20:45:11.657 INFO:teuthology.task.ansible.out: TASK [testnode : Write /scratch_devs] ****************************************** 2021-11-11T20:45:11.659 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => (item={'key': 'lv_4', 'value': {'vg': 'vg_nvme', 'size': '89.4G', 'scratch_dev': True}}) => {"ansible_loop_var": "item", "backup": "", "changed": true, "item": {"key": "lv_4", "value": {"scratch_dev": true, "size": "89.4G", "vg": "vg_nvme"}}, "msg": "line added"} skipping: [smithi198.front.sepia.ceph.com] => (item={'key': 'lv_5', 'value': {'vg': 'vg_nvme', 'size': '14.9G'}}) => {"ansible_loop_var": "item", "changed": false, "item": {"key": "lv_5", "value": {"size": "14.9G", "vg": "vg_nvme"}}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:11.686 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:11 +0000 (0:00:00.037) 0:07:31.942 ***** 2021-11-11T20:45:11.713 INFO:teuthology.task.ansible.out: TASK [testnode : Create filesystems] ******************************************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:11.736 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:11 +0000 (0:00:00.048) 0:07:31.991 ***** 2021-11-11T20:45:11.773 INFO:teuthology.task.ansible.out: TASK [testnode : Mount filesystems] ******************************************** 2021-11-11T20:45:11.775 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:11.797 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:11 +0000 (0:00:00.061) 0:07:32.053 ***** 2021-11-11T20:45:11.856 INFO:teuthology.task.ansible.out: TASK [testnode : Create /var/lib/ceph] ***************************************** 2021-11-11T20:45:11.858 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/var/lib/ceph", "secontext": "unconfined_u:object_r:var_lib_t:s0", "size": 4096, "state": "directory", "uid": 0} 2021-11-11T20:45:11.880 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:11 +0000 (0:00:00.083) 0:07:32.136 ***** 2021-11-11T20:45:11.920 INFO:teuthology.task.ansible.out: TASK [testnode : Set xfs_opts on newer OSes] *********************************** 2021-11-11T20:45:11.921 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:11.946 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:11 +0000 (0:00:00.065) 0:07:32.201 ***** 2021-11-11T20:45:11.996 INFO:teuthology.task.ansible.out: TASK [testnode : Create /var/lib/ceph] ***************************************** 2021-11-11T20:45:11.998 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "gid": 0, "group": "root", "mode": "0755", "owner": "root", "path": "/var/lib/ceph", "secontext": "unconfined_u:object_r:var_lib_t:s0", "size": 4096, "state": "directory", "uid": 0} 2021-11-11T20:45:12.022 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.076) 0:07:32.278 ***** 2021-11-11T20:45:12.060 INFO:teuthology.task.ansible.out: TASK [testnode : Set xfs_opts on newer OSes] *********************************** 2021-11-11T20:45:12.061 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:12.082 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.060) 0:07:32.338 ***** 2021-11-11T20:45:12.309 INFO:teuthology.task.ansible.out: TASK [testnode : Create xfs filesystem on /dev/vg_nvme/lv_5] ******************* 2021-11-11T20:45:12.311 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true} 2021-11-11T20:45:12.335 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.252) 0:07:32.591 ***** 2021-11-11T20:45:12.374 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true} 2021-11-11T20:45:12.400 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.064) 0:07:32.656 ***** 2021-11-11T20:45:12.725 INFO:teuthology.task.ansible.out: TASK [testnode : Mount /dev/vg_nvme/lv_5 to /var/lib/ceph] ********************* 2021-11-11T20:45:12.727 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "dump": "0", "fstab": "/etc/fstab", "fstype": "xfs", "name": "/var/lib/ceph", "opts": "defaults,nofail", "passno": "0", "src": "/dev/vg_nvme/lv_5"} changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "dump": "0", "fstab": "/etc/fstab", "fstype": "xfs", "name": "/var/lib/ceph", "opts": "defaults,nofail", "passno": "0", "src": "/dev/vg_nvme/lv_5"} 2021-11-11T20:45:12.748 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.347) 0:07:33.004 ***** 2021-11-11T20:45:12.769 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.020) 0:07:33.025 ***** 2021-11-11T20:45:12.776 INFO:teuthology.task.ansible.out: TASK [testnode : Install cachefilesd] ****************************************** 2021-11-11T20:45:12.779 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:12.801 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.031) 0:07:33.056 ***** 2021-11-11T20:45:12.810 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:12.832 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.031) 0:07:33.087 ***** 2021-11-11T20:45:12.839 INFO:teuthology.task.ansible.out: TASK [testnode : Install cachefilesd conf file] ******************************** 2021-11-11T20:45:12.841 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:12.864 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.032) 0:07:33.120 ***** 2021-11-11T20:45:12.871 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:12.895 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.030) 0:07:33.151 ***** 2021-11-11T20:45:12.901 INFO:teuthology.task.ansible.out: TASK [testnode : Restart cachefilesd] ****************************************** 2021-11-11T20:45:12.903 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:12.926 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.031) 0:07:33.182 ***** 2021-11-11T20:45:12.934 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:12.958 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.031) 0:07:33.214 ***** 2021-11-11T20:45:12.964 INFO:teuthology.task.ansible.out: TASK [testnode : Restart cachefilesd] ****************************************** 2021-11-11T20:45:12.965 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:12.990 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:12 +0000 (0:00:00.032) 0:07:33.246 ***** 2021-11-11T20:45:12.999 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.022 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.032) 0:07:33.278 ***** 2021-11-11T20:45:13.030 INFO:teuthology.task.ansible.out: TASK [testnode : Add CPAN config directory for the teuthology user.] *********** 2021-11-11T20:45:13.031 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.054 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.031) 0:07:33.310 ***** 2021-11-11T20:45:13.062 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.086 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.031) 0:07:33.342 ***** 2021-11-11T20:45:13.092 INFO:teuthology.task.ansible.out: TASK [testnode : Add CPAN config directory for the root user.] ***************** 2021-11-11T20:45:13.094 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.116 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.030) 0:07:33.372 ***** 2021-11-11T20:45:13.124 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.147 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.030) 0:07:33.403 ***** 2021-11-11T20:45:13.153 INFO:teuthology.task.ansible.out: TASK [testnode : Upload CPAN config for the teuthology user.] ****************** 2021-11-11T20:45:13.155 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.179 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.031) 0:07:33.435 ***** 2021-11-11T20:45:13.187 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.213 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.034) 0:07:33.469 ***** 2021-11-11T20:45:13.221 INFO:teuthology.task.ansible.out: TASK [testnode : Upload CPAN config for root.] ********************************* 2021-11-11T20:45:13.223 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.246 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.032) 0:07:33.502 ***** 2021-11-11T20:45:13.254 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.279 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.032) 0:07:33.535 ***** 2021-11-11T20:45:13.287 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure perl-doc is installed on apt systems.] ***************** 2021-11-11T20:45:13.289 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.314 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.035) 0:07:33.570 ***** 2021-11-11T20:45:13.334 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.360 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.046) 0:07:33.616 ***** 2021-11-11T20:45:13.368 INFO:teuthology.task.ansible.out: TASK [testnode : Check to see if Amazon::S3 is installed.] ********************* 2021-11-11T20:45:13.370 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.400 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.037) 0:07:33.654 ***** 2021-11-11T20:45:13.406 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.432 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.033) 0:07:33.688 ***** 2021-11-11T20:45:13.439 INFO:teuthology.task.ansible.out: TASK [testnode : Install Amazon::S3.] ****************************************** 2021-11-11T20:45:13.441 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.481 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.049) 0:07:33.737 ***** 2021-11-11T20:45:13.491 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.517 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.036) 0:07:33.773 ***** 2021-11-11T20:45:13.524 INFO:teuthology.task.ansible.out: TASK [testnode : Install ntp package on rpm based systems.] ******************** 2021-11-11T20:45:13.526 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.550 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.032) 0:07:33.806 ***** 2021-11-11T20:45:13.568 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.595 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.044) 0:07:33.850 ***** 2021-11-11T20:45:13.605 INFO:teuthology.task.ansible.out: TASK [testnode : Install ntp package on apt based systems.] ******************** 2021-11-11T20:45:13.607 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.641 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.046) 0:07:33.897 ***** 2021-11-11T20:45:13.652 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.677 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.035) 0:07:33.933 ***** 2021-11-11T20:45:13.684 INFO:teuthology.task.ansible.out: TASK [testnode : Remove racy /etc/network/if-up.d/ntpdate on xenial] *********** 2021-11-11T20:45:13.686 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.713 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.035) 0:07:33.969 ***** 2021-11-11T20:45:13.722 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.751 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.038) 0:07:34.007 ***** 2021-11-11T20:45:13.762 INFO:teuthology.task.ansible.out: TASK [testnode : Create the ntp.conf file.] ************************************ 2021-11-11T20:45:13.764 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.789 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.037) 0:07:34.045 ***** 2021-11-11T20:45:13.798 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:13.821 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:13 +0000 (0:00:00.031) 0:07:34.077 ***** 2021-11-11T20:45:14.270 INFO:teuthology.task.ansible.out: TASK [testnode : Create the chrony.conf file] ********************************** ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "a7ecef00073feab34f868065d34e9dac55134241", "dest": "/etc/chrony.conf", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/chrony.conf", "secontext": "system_u:object_r:etc_t:s0", "size": 215, "state": "file", "uid": 0} 2021-11-11T20:45:14.290 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.469) 0:07:34.546 ***** 2021-11-11T20:45:14.319 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "a7ecef00073feab34f868065d34e9dac55134241", "dest": "/etc/chrony.conf", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "path": "/etc/chrony.conf", "secontext": "system_u:object_r:etc_t:s0", "size": 215, "state": "file", "uid": 0} 2021-11-11T20:45:14.343 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.052) 0:07:34.599 ***** 2021-11-11T20:45:14.597 INFO:teuthology.task.ansible.out: TASK [testnode : Make sure ntpd is running.] *********************************** 2021-11-11T20:45:14.599 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "enabled": true, "name": "chronyd", "state": "started", "status": {"ActiveEnterTimestamp": "Thu 2021-11-11 20:36:21 UTC", "ActiveEnterTimestampMonotonic": "15559235", "ActiveExitTimestampMonotonic": "0", "ActiveState": "active", "After": "basic.target tmp.mount sntp.service systemd-tmpfiles-setup.service ntpdate.service systemd-journald.socket sysinit.target ntpd.service -.mount system.slice", "AllowIsolate": "no", "AllowedCPUs": "", "AllowedMemoryNodes": "", "AmbientCapabilities": "", "AssertResult": "yes", "AssertTimestamp": "Thu 2021-11-11 20:36:19 UTC", "AssertTimestampMonotonic": "13976707", "Before": "multi-user.target shutdown.target", "BlockIOAccounting": "no", "BlockIOWeight": "[not set]", "CPUAccounting": "no", "CPUAffinity": "", "CPUAffinityFromNUMA": "no", "CPUQuotaPerSecUSec": "infinity", "CPUQuotaPeriodUSec": "infinity", "CPUSchedulingPolicy": "0", "CPUSchedulingPriority": "0", "CPUSchedulingResetOnFork": "no", "CPUShares": "[not set]", "CPUUsageNSec": "[not set]", "CPUWeight": "[not set]", "CacheDirectoryMode": "0755", "CanFreeze": "yes", "CanIsolate": "no", "CanReload": "no", "CanStart": "yes", "CanStop": "yes", "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon", "CollectMode": "inactive", "ConditionResult": "yes", "ConditionTimestamp": "Thu 2021-11-11 20:36:19 UTC", "ConditionTimestampMonotonic": "13976677", "ConfigurationDirectoryMode": "0755", "Conflicts": "ntpd.service 2021-11-11T20:45:14.602 INFO:teuthology.task.ansible.out: shutdown.target systemd-timesyncd.service", "ControlGroup": "/system.slice/chronyd.service", "ControlPID": "0", "DefaultDependencies": "yes", "DefaultMemoryLow": "0", "DefaultMemoryMin": "0", "Delegate": "no", "Description": "NTP client/server", "DevicePolicy": "auto", "Documentation": "man:chronyd(8) man:chrony.conf(5)", "DynamicUser": "no", "EffectiveCPUs": "", "EffectiveMemoryNodes": "", "EnvironmentFiles": "/etc/sysconfig/chronyd (ignore_errors=yes)", "ExecMainCode": "0", "ExecMainExitTimestampMonotonic": "0", "ExecMainPID": "867", "ExecMainStartTimestamp": "Thu 2021-11-11 20:36:20 UTC", "ExecMainStartTimestampMonotonic": "15382364", "ExecMainStatus": "0", "ExecStart": "{ path=/usr/sbin/chronyd ; argv[]=/usr/sbin/chronyd $OPTIONS ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "ExecStartPost": "{ path=/usr/libexec/chrony-helper ; argv[]=/usr/libexec/chrony-helper update-daemon ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "FailureAction": "none", "FileDescriptorStoreMax": "0", "FragmentPath": "/usr/lib/systemd/system/chronyd.service", "FreezerState": "running", "GID": "[not set]", "GuessMainPID": "yes", "IOAccounting": "no", "IOSchedulingClass": "0", "IOSchedulingPriority": "0", "IOWeight": "[not set]", "IPAccounting": "no", "IPEgressBytes": "18446744073709551615", "IPEgressPackets": "18446744073709551615", "IPIngressBytes": "18446744073709551615", "IPIngressPackets": "18446744073709551615", "Id": "chronyd.service", "IgnoreOnIsolate": "no", "IgnoreSIGPIPE": "yes", "InactiveEnterTimestampMonotonic": "0", "InactiveExitTimestamp": "Thu 2021-11-11 20:36:19 UTC", "InactiveExitTimestampMonotonic": "13985510", "InvocationID": "075c7d6e763e458b89079038a888d426", "JobRunningTimeoutUSec": "infinity", "JobTimeoutAction": "none", "JobTimeoutUSec": "infinity", "KeyringMode": "private", "KillMode": "control-group", "KillSignal": "15", "LimitAS": "infinity", "LimitASSoft": "infinit 2021-11-11T20:45:14.604 INFO:teuthology.task.ansible.out:y", "LimitCORE": "infinity", "LimitCORESoft": "infinity", "LimitCPU": "infinity", "LimitCPUSoft": "infinity", "LimitDATA": "infinity", "LimitDATASoft": "infinity", "LimitFSIZE": "infinity", "LimitFSIZESoft": "infinity", "LimitLOCKS": "infinity", "LimitLOCKSSoft": "infinity", "LimitMEMLOCK": "65536", "LimitMEMLOCKSoft": "65536", "LimitMSGQUEUE": "819200", "LimitMSGQUEUESoft": "819200", "LimitNICE": "0", "LimitNICESoft": "0", "LimitNOFILE": "262144", "LimitNOFILESoft": "1024", "LimitNPROC": "127240", "LimitNPROCSoft": "127240", "LimitRSS": "infinity", "LimitRSSSoft": "infinity", "LimitRTPRIO": "0", "LimitRTPRIOSoft": "0", "LimitRTTIME": "infinity", "LimitRTTIMESoft": "infinity", "LimitSIGPENDING": "127240", "LimitSIGPENDINGSoft": "127240", "LimitSTACK": "infinity", "LimitSTACKSoft": "8388608", "LoadState": "loaded", "LockPersonality": "no", "LogLevelMax": "-1", "LogRateLimitBurst": "0", "LogRateLimitIntervalUSec": "0", "LogsDirectoryMode": "0755", "MainPID": "867", "MemoryAccounting": "yes", "MemoryCurrent": "1253376", "MemoryDenyWriteExecute": "no", "MemoryHigh": "infinity", "MemoryLimit": "infinity", "MemoryLow": "0", "MemoryMax": "infinity", "MemoryMin": "0", "MemorySwapMax": "infinity", "MountAPIVFS": "no", "MountFlags": "", "NFileDescriptorStore": "0", "NRestarts": "0", "NUMAMask": "", "NUMAPolicy": "n/a", "Names": "chronyd.service", "NeedDaemonReload": "no", "Nice": "0", "NoNewPrivileges": "no", "NonBlocking": "no", "NotifyAccess": "none", "OOMScoreAdjust": "0", "OnFailureJobMode": "replace", "PIDFile": "/run/chrony/chronyd.pid", "PermissionsStartOnly": "no", "Perpetual": "no", "PrivateDevices": "no", "PrivateMounts": "no", "PrivateNetwork": "no", "PrivateTmp": "yes", "PrivateUsers": "no", "ProtectControlGroups": "no", "ProtectHome": "yes", "ProtectKernelModules": "no", "ProtectKernelTunables": "no", "ProtectSystem": "full", "RefuseManualStart": "no", "RefuseManualStop": "no", "RemainAfterExit": "no", "RemoveIPC": "no", "Requires": "sysinit.target -.mount system 2021-11-11T20:45:14.606 INFO:teuthology.task.ansible.out:.slice", "RequiresMountsFor": "/var/tmp", "Restart": "no", "RestartUSec": "100ms", "RestrictNamespaces": "no", "RestrictRealtime": "no", "RestrictSUIDSGID": "no", "Result": "success", "RootDirectoryStartOnly": "no", "RuntimeDirectoryMode": "0755", "RuntimeDirectoryPreserve": "no", "RuntimeMaxUSec": "infinity", "SameProcessGroup": "no", "SecureBits": "0", "SendSIGHUP": "no", "SendSIGKILL": "yes", "Slice": "system.slice", "StandardError": "inherit", "StandardInput": "null", "StandardInputData": "", "StandardOutput": "journal", "StartLimitAction": "none", "StartLimitBurst": "5", "StartLimitIntervalUSec": "10s", "StartupBlockIOWeight": "[not set]", "StartupCPUShares": "[not set]", "StartupCPUWeight": "[not set]", "StartupIOWeight": "[not set]", "StateChangeTimestamp": "Thu 2021-11-11 20:36:21 UTC", "StateChangeTimestampMonotonic": "15559235", "StateDirectoryMode": "0755", "StatusErrno": "0", "StopWhenUnneeded": "no", "SubState": "running", "SuccessAction": "none", "SyslogFacility": "3", "SyslogLevel": "6", "SyslogLevelPrefix": "yes", "SyslogPriority": "30", "SystemCallErrorNumber": "0", "TTYReset": "no", "TTYVHangup": "no", "TTYVTDisallocate": "no", "TasksAccounting": "yes", "TasksCurrent": "1", "TasksMax": "203584", "TimeoutStartUSec": "1min 30s", "TimeoutStopUSec": "1min 30s", "TimerSlackNSec": "50000", "Transient": "no", "Type": "forking", "UID": "[not set]", "UMask": "0022", "UnitFilePreset": "enabled", "UnitFileState": "enabled", "UtmpMode": "init", "WantedBy": "multi-user.target", "WatchdogTimestamp": "Thu 2021-11-11 20:36:20 UTC", "WatchdogTimestampMonotonic": "15382375", "WatchdogUSec": "0"}} 2021-11-11T20:45:14.623 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.280) 0:07:34.879 ***** 2021-11-11T20:45:14.653 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "enabled": true, "name": "chronyd", "state": "started", "status": {"ActiveEnterTimestamp": "Thu 2021-11-11 20:36:13 UTC", "ActiveEnterTimestampMonotonic": "15223119", "ActiveExitTimestampMonotonic": "0", "ActiveState": "active", "After": "sntp.service tmp.mount ntpdate.service sysinit.target ntpd.service systemd-journald.socket systemd-tmpfiles-setup.service basic.target system.slice -.mount", "AllowIsolate": "no", "AllowedCPUs": "", "AllowedMemoryNodes": "", "AmbientCapabilities": "", "AssertResult": "yes", "AssertTimestamp": "Thu 2021-11-11 20:36:12 UTC", "AssertTimestampMonotonic": "13734473", "Before": "multi-user.target shutdown.target", "BlockIOAccounting": "no", "BlockIOWeight": "[not set]", "CPUAccounting": "no", "CPUAffinity": "", "CPUAffinityFromNUMA": "no", "CPUQuotaPerSecUSec": "infinity", "CPUQuotaPeriodUSec": "infinity", "CPUSchedulingPolicy": "0", "CPUSchedulingPriority": "0", "CPUSchedulingResetOnFork": "no", "CPUShares": "[not set]", "CPUUsageNSec": "[not set]", "CPUWeight": "[not set]", "CacheDirectoryMode": "0755", "CanFreeze": "yes", "CanIsolate": "no", "CanReload": "no", "CanStart": "yes", "CanStop": "yes", "CapabilityBoundingSet": "cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend cap_audit_read cap_perfmon", "CollectMode": "inactive", "ConditionResult": "yes", "ConditionTimestamp": "Thu 2021-11-11 20:36:12 UTC", "ConditionTimestampMonotonic": "13734434", "ConfigurationDirectoryMode": "0755", "Conflicts": "systemd-time 2021-11-11T20:45:14.655 INFO:teuthology.task.ansible.out:syncd.service ntpd.service shutdown.target", "ControlGroup": "/system.slice/chronyd.service", "ControlPID": "0", "DefaultDependencies": "yes", "DefaultMemoryLow": "0", "DefaultMemoryMin": "0", "Delegate": "no", "Description": "NTP client/server", "DevicePolicy": "auto", "Documentation": "man:chronyd(8) man:chrony.conf(5)", "DynamicUser": "no", "EffectiveCPUs": "", "EffectiveMemoryNodes": "", "EnvironmentFiles": "/etc/sysconfig/chronyd (ignore_errors=yes)", "ExecMainCode": "0", "ExecMainExitTimestampMonotonic": "0", "ExecMainPID": "855", "ExecMainStartTimestamp": "Thu 2021-11-11 20:36:12 UTC", "ExecMainStartTimestampMonotonic": "14058172", "ExecMainStatus": "0", "ExecStart": "{ path=/usr/sbin/chronyd ; argv[]=/usr/sbin/chronyd $OPTIONS ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "ExecStartPost": "{ path=/usr/libexec/chrony-helper ; argv[]=/usr/libexec/chrony-helper update-daemon ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }", "FailureAction": "none", "FileDescriptorStoreMax": "0", "FragmentPath": "/usr/lib/systemd/system/chronyd.service", "FreezerState": "running", "GID": "[not set]", "GuessMainPID": "yes", "IOAccounting": "no", "IOSchedulingClass": "0", "IOSchedulingPriority": "0", "IOWeight": "[not set]", "IPAccounting": "no", "IPEgressBytes": "18446744073709551615", "IPEgressPackets": "18446744073709551615", "IPIngressBytes": "18446744073709551615", "IPIngressPackets": "18446744073709551615", "Id": "chronyd.service", "IgnoreOnIsolate": "no", "IgnoreSIGPIPE": "yes", "InactiveEnterTimestampMonotonic": "0", "InactiveExitTimestamp": "Thu 2021-11-11 20:36:12 UTC", "InactiveExitTimestampMonotonic": "13762165", "InvocationID": "0b15d6074a784619ae0d656da58c58b7", "JobRunningTimeoutUSec": "infinity", "JobTimeoutAction": "none", "JobTimeoutUSec": "infinity", "KeyringMode": "private", "KillMode": "control-group", "KillSignal": "15", "LimitAS": "infinity", "LimitASSoft": "infinit 2021-11-11T20:45:14.657 INFO:teuthology.task.ansible.out:y", "LimitCORE": "infinity", "LimitCORESoft": "infinity", "LimitCPU": "infinity", "LimitCPUSoft": "infinity", "LimitDATA": "infinity", "LimitDATASoft": "infinity", "LimitFSIZE": "infinity", "LimitFSIZESoft": "infinity", "LimitLOCKS": "infinity", "LimitLOCKSSoft": "infinity", "LimitMEMLOCK": "65536", "LimitMEMLOCKSoft": "65536", "LimitMSGQUEUE": "819200", "LimitMSGQUEUESoft": "819200", "LimitNICE": "0", "LimitNICESoft": "0", "LimitNOFILE": "262144", "LimitNOFILESoft": "1024", "LimitNPROC": "127240", "LimitNPROCSoft": "127240", "LimitRSS": "infinity", "LimitRSSSoft": "infinity", "LimitRTPRIO": "0", "LimitRTPRIOSoft": "0", "LimitRTTIME": "infinity", "LimitRTTIMESoft": "infinity", "LimitSIGPENDING": "127240", "LimitSIGPENDINGSoft": "127240", "LimitSTACK": "infinity", "LimitSTACKSoft": "8388608", "LoadState": "loaded", "LockPersonality": "no", "LogLevelMax": "-1", "LogRateLimitBurst": "0", "LogRateLimitIntervalUSec": "0", "LogsDirectoryMode": "0755", "MainPID": "855", "MemoryAccounting": "yes", "MemoryCurrent": "1249280", "MemoryDenyWriteExecute": "no", "MemoryHigh": "infinity", "MemoryLimit": "infinity", "MemoryLow": "0", "MemoryMax": "infinity", "MemoryMin": "0", "MemorySwapMax": "infinity", "MountAPIVFS": "no", "MountFlags": "", "NFileDescriptorStore": "0", "NRestarts": "0", "NUMAMask": "", "NUMAPolicy": "n/a", "Names": "chronyd.service", "NeedDaemonReload": "no", "Nice": "0", "NoNewPrivileges": "no", "NonBlocking": "no", "NotifyAccess": "none", "OOMScoreAdjust": "0", "OnFailureJobMode": "replace", "PIDFile": "/run/chrony/chronyd.pid", "PermissionsStartOnly": "no", "Perpetual": "no", "PrivateDevices": "no", "PrivateMounts": "no", "PrivateNetwork": "no", "PrivateTmp": "yes", "PrivateUsers": "no", "ProtectControlGroups": "no", "ProtectHome": "yes", "ProtectKernelModules": "no", "ProtectKernelTunables": "no", "ProtectSystem": "full", "RefuseManualStart": "no", "RefuseManualStop": "no", "RemainAfterExit": "no", "RemoveIPC": "no", "Requires": "system.slice -.mount sysinit. 2021-11-11T20:45:14.659 INFO:teuthology.task.ansible.out:target", "RequiresMountsFor": "/var/tmp", "Restart": "no", "RestartUSec": "100ms", "RestrictNamespaces": "no", "RestrictRealtime": "no", "RestrictSUIDSGID": "no", "Result": "success", "RootDirectoryStartOnly": "no", "RuntimeDirectoryMode": "0755", "RuntimeDirectoryPreserve": "no", "RuntimeMaxUSec": "infinity", "SameProcessGroup": "no", "SecureBits": "0", "SendSIGHUP": "no", "SendSIGKILL": "yes", "Slice": "system.slice", "StandardError": "inherit", "StandardInput": "null", "StandardInputData": "", "StandardOutput": "journal", "StartLimitAction": "none", "StartLimitBurst": "5", "StartLimitIntervalUSec": "10s", "StartupBlockIOWeight": "[not set]", "StartupCPUShares": "[not set]", "StartupCPUWeight": "[not set]", "StartupIOWeight": "[not set]", "StateChangeTimestamp": "Thu 2021-11-11 20:36:13 UTC", "StateChangeTimestampMonotonic": "15223119", "StateDirectoryMode": "0755", "StatusErrno": "0", "StopWhenUnneeded": "no", "SubState": "running", "SuccessAction": "none", "SyslogFacility": "3", "SyslogLevel": "6", "SyslogLevelPrefix": "yes", "SyslogPriority": "30", "SystemCallErrorNumber": "0", "TTYReset": "no", "TTYVHangup": "no", "TTYVTDisallocate": "no", "TasksAccounting": "yes", "TasksCurrent": "1", "TasksMax": "203584", "TimeoutStartUSec": "1min 30s", "TimeoutStopUSec": "1min 30s", "TimerSlackNSec": "50000", "Transient": "no", "Type": "forking", "UID": "[not set]", "UMask": "0022", "UnitFilePreset": "enabled", "UnitFileState": "enabled", "UtmpMode": "init", "WantedBy": "multi-user.target", "WatchdogTimestamp": "Thu 2021-11-11 20:36:12 UTC", "WatchdogTimestampMonotonic": "14058182", "WatchdogUSec": "0"}} TASK [testnode : set_fact] ***************************************************** ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"pip_executable": "pip", "pip_version": "python-pip"}, "changed": false} 2021-11-11T20:45:14.677 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.054) 0:07:34.933 ***** 2021-11-11T20:45:14.701 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.022) 0:07:34.956 ***** 2021-11-11T20:45:14.719 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"ansible_python_interpreter": "/usr/bin/python3", "pip_executable": "pip3", "pip_version": "python3-pip"}, "changed": false} 2021-11-11T20:45:14.742 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.042) 0:07:34.998 ***** 2021-11-11T20:45:14.754 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** 2021-11-11T20:45:14.756 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"pip_executable": "pip", "pip_version": "python-pip"}, "changed": false} 2021-11-11T20:45:14.782 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.039) 0:07:35.038 ***** 2021-11-11T20:45:14.790 INFO:teuthology.task.ansible.out: TASK [testnode : Install python-pip on rpm based systems.] ********************* 2021-11-11T20:45:14.792 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:14.815 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.032) 0:07:35.070 ***** 2021-11-11T20:45:14.823 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"ansible_python_interpreter": "/usr/bin/python3", "pip_executable": "pip3", "pip_version": "python3-pip"}, "changed": false} 2021-11-11T20:45:14.854 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.037) 0:07:35.108 ***** 2021-11-11T20:45:14.861 INFO:teuthology.task.ansible.out: TASK [testnode : Install python-pip on apt based systems.] ********************* skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:14.885 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.033) 0:07:35.141 ***** 2021-11-11T20:45:14.895 INFO:teuthology.task.ansible.out: TASK [testnode : Install python-pip on rpm based systems.] ********************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:14.922 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.037) 0:07:35.178 ***** 2021-11-11T20:45:14.929 INFO:teuthology.task.ansible.out: TASK [testnode : Install python-pip on zypper based systems.] ****************** 2021-11-11T20:45:14.931 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:14.954 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.031) 0:07:35.209 ***** 2021-11-11T20:45:14.962 INFO:teuthology.task.ansible.out: TASK [testnode : Install python-pip on apt based systems.] ********************* 2021-11-11T20:45:14.963 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:14.990 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:14 +0000 (0:00:00.034) 0:07:35.244 ***** 2021-11-11T20:45:15.017 INFO:teuthology.task.ansible.out: TASK [testnode : Install python-pip on zypper based systems.] ****************** 2021-11-11T20:45:15.019 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:15.042 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:15 +0000 (0:00:00.053) 0:07:35.298 ***** 2021-11-11T20:45:15.154 INFO:teuthology.task.ansible.out: TASK [testnode : Create the .pip directory for the teuthology user.] *********** 2021-11-11T20:45:15.155 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "gid": 1238, "group": "ubuntu", "mode": "0644", "owner": "ubuntu", "path": "/home/ubuntu/.pip", "secontext": "unconfined_u:object_r:user_home_t:s0", "size": 4096, "state": "directory", "uid": 1000} 2021-11-11T20:45:15.178 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:15 +0000 (0:00:00.136) 0:07:35.434 ***** 2021-11-11T20:45:15.240 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "gid": 1238, "group": "ubuntu", "mode": "0644", "owner": "ubuntu", "path": "/home/ubuntu/.pip", "secontext": "unconfined_u:object_r:user_home_t:s0", "size": 4096, "state": "directory", "uid": 1000} 2021-11-11T20:45:15.266 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:15 +0000 (0:00:00.086) 0:07:35.521 ***** 2021-11-11T20:45:15.649 INFO:teuthology.task.ansible.out: TASK [testnode : Create pip.conf and configure it to use our mirror] *********** 2021-11-11T20:45:15.651 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "checksum": "959d37ab30545da205b389a6b7561b225a478a06", "dest": "/home/ubuntu/.pip/pip.conf", "gid": 1238, "group": "ubuntu", "mode": "0644", "owner": "ubuntu", "path": "/home/ubuntu/.pip/pip.conf", "secontext": "unconfined_u:object_r:user_home_t:s0", "size": 73, "state": "file", "uid": 1000} 2021-11-11T20:45:15.672 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:15 +0000 (0:00:00.407) 0:07:35.928 ***** 2021-11-11T20:45:15.734 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "checksum": "959d37ab30545da205b389a6b7561b225a478a06", "dest": "/home/ubuntu/.pip/pip.conf", "gid": 1238, "group": "ubuntu", "mode": "0644", "owner": "ubuntu", "path": "/home/ubuntu/.pip/pip.conf", "secontext": "unconfined_u:object_r:user_home_t:s0", "size": 73, "state": "file", "uid": 1000} 2021-11-11T20:45:15.758 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:15 +0000 (0:00:00.085) 0:07:36.014 ***** 2021-11-11T20:45:16.817 INFO:teuthology.task.ansible.out: TASK [testnode : Install packages via pip] ************************************* 2021-11-11T20:45:16.819 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "cmd": ["/bin/pip3", "install", "remoto>=0.0.35"], "name": ["remoto>=0.0.35"], "requirements": null, "state": "present", "stderr": "WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead.\n", "stderr_lines": ["WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead."], "stdout": "Requirement already satisfied: remoto>=0.0.35 in /usr/local/lib/python3.6/site-packages\nRequirement already satisfied: execnet in /usr/local/lib/python3.6/site-packages (from remoto>=0.0.35)\nRequirement already satisfied: apipkg>=1.4 in /usr/local/lib/python3.6/site-packages (from execnet->remoto>=0.0.35)\n", "stdout_lines": ["Requirement already satisfied: remoto>=0.0.35 in /usr/local/lib/python3.6/site-packages", "Requirement already satisfied: execnet in /usr/local/lib/python3.6/site-packages (from remoto>=0.0.35)", "Requirement already satisfied: apipkg>=1.4 in /usr/local/lib/python3.6/site-packages (from execnet->remoto>=0.0.35)"], "version": null, "virtualenv": null} 2021-11-11T20:45:16.840 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:16 +0000 (0:00:01.081) 0:07:37.096 ***** 2021-11-11T20:45:16.848 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "cmd": ["/bin/pip3", "install", "remoto>=0.0.35"], "name": ["remoto>=0.0.35"], "requirements": null, "state": "present", "stderr": "WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead.\n", "stderr_lines": ["WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead."], "stdout": "Requirement already satisfied: remoto>=0.0.35 in /usr/local/lib/python3.6/site-packages\nRequirement already satisfied: execnet in /usr/local/lib/python3.6/site-packages (from remoto>=0.0.35)\nRequirement already satisfied: apipkg>=1.4 in /usr/local/lib/python3.6/site-packages (from execnet->remoto>=0.0.35)\n", "stdout_lines": ["Requirement already satisfied: remoto>=0.0.35 in /usr/local/lib/python3.6/site-packages", "Requirement already satisfied: execnet in /usr/local/lib/python3.6/site-packages (from remoto>=0.0.35)", "Requirement already satisfied: apipkg>=1.4 in /usr/local/lib/python3.6/site-packages (from execnet->remoto>=0.0.35)"], "version": null, "virtualenv": null} 2021-11-11T20:45:16.874 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:16 +0000 (0:00:00.033) 0:07:37.130 ***** 2021-11-11T20:45:16.881 INFO:teuthology.task.ansible.out: TASK [testnode : Purge resolvconf] ********************************************* 2021-11-11T20:45:16.883 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:16.910 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:16 +0000 (0:00:00.035) 0:07:37.165 ***** 2021-11-11T20:45:16.917 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:16.941 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:16 +0000 (0:00:00.031) 0:07:37.197 ***** 2021-11-11T20:45:16.948 INFO:teuthology.task.ansible.out: TASK [testnode : Set interface] ************************************************ 2021-11-11T20:45:16.950 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"interface": "enp3s0f1"}, "changed": false} 2021-11-11T20:45:16.978 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:16 +0000 (0:00:00.033) 0:07:37.230 ***** 2021-11-11T20:45:16.982 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"interface": "enp3s0f1"}, "changed": false} 2021-11-11T20:45:17.010 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.032) 0:07:37.263 ***** 2021-11-11T20:45:17.160 INFO:teuthology.task.ansible.out: TASK [testnode : Check for presence of /etc/network/interfaces] **************** 2021-11-11T20:45:17.162 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "stat": {"exists": false}} 2021-11-11T20:45:17.196 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.188) 0:07:37.452 ***** 2021-11-11T20:45:17.205 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "stat": {"exists": false}} 2021-11-11T20:45:17.231 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.034) 0:07:37.487 ***** 2021-11-11T20:45:17.240 INFO:teuthology.task.ansible.out: TASK [testnode : Rewrite /etc/network/interfaces to use dhcp] ****************** 2021-11-11T20:45:17.242 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:17.264 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.033) 0:07:37.520 ***** 2021-11-11T20:45:17.272 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:17.296 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.031) 0:07:37.552 ***** 2021-11-11T20:45:17.306 INFO:teuthology.task.ansible.out: TASK [testnode : Set bounce_interface if we just enabled dhcp] ***************** 2021-11-11T20:45:17.308 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"bounce_interface": false}, "changed": false} 2021-11-11T20:45:17.329 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.033) 0:07:37.585 ***** 2021-11-11T20:45:17.335 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"bounce_interface": false}, "changed": false} 2021-11-11T20:45:17.361 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.031) 0:07:37.617 ***** 2021-11-11T20:45:17.368 INFO:teuthology.task.ansible.out: TASK [testnode : ifdown and ifup] ********************************************** 2021-11-11T20:45:17.370 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:17.391 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.030) 0:07:37.647 ***** 2021-11-11T20:45:17.403 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:17.428 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.037) 0:07:37.684 ***** 2021-11-11T20:45:17.584 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure lab_domain is in search domains in /etc/resolv.conf] *** 2021-11-11T20:45:17.586 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"backup": "", "changed": true, "msg": "line replaced"} 2021-11-11T20:45:17.610 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.181) 0:07:37.866 ***** 2021-11-11T20:45:17.629 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"backup": "", "changed": true, "msg": "line replaced"} 2021-11-11T20:45:17.656 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.046) 0:07:37.912 ***** 2021-11-11T20:45:17.803 INFO:teuthology.task.ansible.out: TASK [testnode : Ensure domain is set in /etc/resolv.conf] ********************* 2021-11-11T20:45:17.806 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"backup": "", "changed": true, "msg": "line added"} 2021-11-11T20:45:17.826 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.170) 0:07:38.082 ***** 2021-11-11T20:45:17.855 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"backup": "", "changed": true, "msg": "line added"} 2021-11-11T20:45:17.857 INFO:teuthology.task.ansible.out: TASK [testnode : Make sure ethtool is installed] ******************************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:17.878 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.051) 0:07:38.134 ***** 2021-11-11T20:45:17.900 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.021) 0:07:38.156 ***** 2021-11-11T20:45:17.907 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:17.933 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.033) 0:07:38.189 ***** 2021-11-11T20:45:17.947 INFO:teuthology.task.ansible.out: TASK [testnode : grep ethtool for secondary NIC MAC address] ******************* 2021-11-11T20:45:17.949 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=enp3s0f0) => {"ansible_loop_var": "item", "changed": false, "item": "enp3s0f0", "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item=eno2) => {"ansible_loop_var": "item", "changed": false, "item": "eno2", "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item=enp3s0f1) => {"ansible_loop_var": "item", "changed": false, "item": "enp3s0f1", "skip_reason": "Conditional result was False"} 2021-11-11T20:45:17.960 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=lo) => {"ansible_loop_var": "item", "changed": false, "item": "lo", "skip_reason": "Conditional result was False"} 2021-11-11T20:45:17.962 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item=eno1) => {"ansible_loop_var": "item", "changed": false, "item": "eno1", "skip_reason": "Conditional result was False"} 2021-11-11T20:45:17.986 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:17 +0000 (0:00:00.052) 0:07:38.242 ***** 2021-11-11T20:45:17.994 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=lo) => {"ansible_loop_var": "item", "changed": false, "item": "lo", "skip_reason": "Conditional result was False"} 2021-11-11T20:45:17.999 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item=eno2) => {"ansible_loop_var": "item", "changed": false, "item": "eno2", "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item=eno1) => {"ansible_loop_var": "item", "changed": false, "item": "eno1", "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item=enp3s0f0) => {"ansible_loop_var": "item", "changed": false, "item": "enp3s0f0", "skip_reason": "Conditional result was False"} skipping: [smithi096.front.sepia.ceph.com] => (item=enp3s0f1) => {"ansible_loop_var": "item", "changed": false, "item": "enp3s0f1", "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.025 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.038) 0:07:38.281 ***** 2021-11-11T20:45:18.026 INFO:teuthology.task.ansible.out: 2021-11-11T20:45:18.037 INFO:teuthology.task.ansible.out: TASK [testnode : Define net_to_configure var] ********************************** 2021-11-11T20:45:18.039 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'enp3s0f0', 'ansible_loop_var': 'item'}) => {"ansible_loop_var": "item", "changed": false, "item": {"ansible_loop_var": "item", "changed": false, "item": "enp3s0f0", "skip_reason": "Conditional result was False", "skipped": true}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.049 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'eno2', 'ansible_loop_var': 'item'}) => {"ansible_loop_var": "item", "changed": false, "item": {"ansible_loop_var": "item", "changed": false, "item": "eno2", "skip_reason": "Conditional result was False", "skipped": true}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.050 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'enp3s0f1', 'ansible_loop_var': 'item'}) => {"ansible_loop_var": "item", "changed": false, "item": {"ansible_loop_var": "item", "changed": false, "item": "enp3s0f1", "skip_reason": "Conditional result was False", "skipped": true}, "skip_reason": "Conditional result was False"} skipping: [smithi198.front.sepia.ceph.com] => (item={'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'lo', 'ansible_loop_var': 'item'}) => {"ansible_loop_var": "item", "changed": false, "item": {"ansible_loop_var": "item", "changed": false, "item": "lo", "skip_reason": "Conditional result was False", "skipped": true}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.061 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => (item={'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'eno1', 'ansible_loop_var': 'item'}) => {"ansible_loop_var": "item", "changed": false, "item": {"ansible_loop_var": "item", "changed": false, "item": "eno1", "skip_reason": "Conditional result was False", "skipped": true}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.089 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.063) 0:07:38.344 ***** 2021-11-11T20:45:18.097 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'lo', 'ansible_loop_var': 'item'}) => {"ansible_loop_var": "item", "changed": false, "item": {"ansible_loop_var": "item", "changed": false, "item": "lo", "skip_reason": "Conditional result was False", "skipped": true}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.098 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'eno2', 'ansible_loop_var': 'item'}) => {"ansible_loop_var": "item", "changed": false, "item": {"ansible_loop_var": "item", "changed": false, "item": "eno2", "skip_reason": "Conditional result was False", "skipped": true}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.100 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'eno1', 'ansible_loop_var': 'item'}) => {"ansible_loop_var": "item", "changed": false, "item": {"ansible_loop_var": "item", "changed": false, "item": "eno1", "skip_reason": "Conditional result was False", "skipped": true}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.111 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'enp3s0f0', 'ansible_loop_var': 'item'}) => {"ansible_loop_var": "item", "changed": false, "item": {"ansible_loop_var": "item", "changed": false, "item": "enp3s0f0", "skip_reason": "Conditional result was False", "skipped": true}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.113 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => (item={'changed': False, 'skipped': True, 'skip_reason': 'Conditional result was False', 'item': 'enp3s0f1', 'ansible_loop_var': 'item'}) => {"ansible_loop_var": "item", "changed": false, "item": {"ansible_loop_var": "item", "changed": false, "item": "enp3s0f1", "skip_reason": "Conditional result was False", "skipped": true}, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.136 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.047) 0:07:38.392 ***** 2021-11-11T20:45:18.143 INFO:teuthology.task.ansible.out: TASK [testnode : set_fact] ***************************************************** 2021-11-11T20:45:18.144 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.170 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.033) 0:07:38.426 ***** 2021-11-11T20:45:18.177 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.202 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.032) 0:07:38.458 ***** 2021-11-11T20:45:18.210 INFO:teuthology.task.ansible.out: TASK [testnode : Check if {{ nic_to_configure }} is 10Gb] ********************** 2021-11-11T20:45:18.211 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.233 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.030) 0:07:38.489 ***** 2021-11-11T20:45:18.240 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.264 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.030) 0:07:38.520 ***** 2021-11-11T20:45:18.272 INFO:teuthology.task.ansible.out: TASK [testnode : Set MTU to 9000 if 10Gb] ************************************** skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.298 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.033) 0:07:38.554 ***** 2021-11-11T20:45:18.305 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.333 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.035) 0:07:38.589 ***** 2021-11-11T20:45:18.341 INFO:teuthology.task.ansible.out: TASK [testnode : Write Ubuntu network config for {{ nic_to_configure }}] ******* 2021-11-11T20:45:18.343 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.367 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.032) 0:07:38.621 ***** 2021-11-11T20:45:18.373 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.398 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.032) 0:07:38.654 ***** 2021-11-11T20:45:18.409 INFO:teuthology.task.ansible.out: TASK [testnode : Set MTU on Ubuntu] ******************************************** skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.433 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.034) 0:07:38.689 ***** 2021-11-11T20:45:18.442 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.474 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.038) 0:07:38.727 ***** 2021-11-11T20:45:18.479 INFO:teuthology.task.ansible.out: TASK [testnode : Bounce {{ nic_to_configure }}] ******************************** 2021-11-11T20:45:18.481 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.507 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.034) 0:07:38.762 ***** 2021-11-11T20:45:18.513 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.540 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.034) 0:07:38.796 ***** 2021-11-11T20:45:18.549 INFO:teuthology.task.ansible.out: TASK [testnode : Write RHEL/CentOS network config for {{ nic_to_configure }}] *** 2021-11-11T20:45:18.550 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.578 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.037) 0:07:38.834 ***** 2021-11-11T20:45:18.586 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.616 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.033) 0:07:38.868 ***** 2021-11-11T20:45:18.620 INFO:teuthology.task.ansible.out: TASK [testnode : Bounce {{ nic_to_configure }}] ******************************** 2021-11-11T20:45:18.621 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.644 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.031) 0:07:38.900 ***** 2021-11-11T20:45:18.652 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:18.676 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:18 +0000 (0:00:00.032) 0:07:38.932 ***** 2021-11-11T20:45:18.861 INFO:teuthology.task.ansible.out: TASK [testnode : List any leftover Ceph artifacts from previous jobs] ********** 2021-11-11T20:45:18.863 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/var/run/) => {"ansible_loop_var": "item", "changed": false, "cmd": "find /var/run/ -name \"*ceph*\"", "delta": "0:00:00.005141", "end": "2021-11-11 20:45:18.828467", "failed_when_result": false, "item": "/var/run/", "rc": 0, "start": "2021-11-11 20:45:18.823326", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:19.030 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/etc/systemd/system/) => {"ansible_loop_var": "item", "changed": false, "cmd": "find /etc/systemd/system/ -name \"*ceph*\"", "delta": "0:00:00.002924", "end": "2021-11-11 20:45:19.001932", "failed_when_result": false, "item": "/etc/systemd/system/", "rc": 0, "start": "2021-11-11 20:45:18.999008", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:19.201 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/etc/ceph) => {"ansible_loop_var": "item", "changed": false, "cmd": "find /etc/ceph -name \"*ceph*\"", "delta": "0:00:00.002934", "end": "2021-11-11 20:45:19.169912", "failed_when_result": false, "item": "/etc/ceph", "msg": "non-zero return code", "rc": 1, "start": "2021-11-11 20:45:19.166978", "stderr": "find: ‘/etc/ceph’: No such file or directory", "stderr_lines": ["find: ‘/etc/ceph’: No such file or directory"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:19.367 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/var/log/) => {"ansible_loop_var": "item", "changed": false, "cmd": "find /var/log/ -name \"*ceph*\"", "delta": "0:00:00.003218", "end": "2021-11-11 20:45:19.344394", "failed_when_result": false, "item": "/var/log/", "rc": 0, "start": "2021-11-11 20:45:19.341176", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:19.400 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:19 +0000 (0:00:00.723) 0:07:39.656 ***** 2021-11-11T20:45:19.428 INFO:teuthology.task.ansible.out: TASK [testnode : Delete any leftover Ceph artifacts from previous jobs] ******** 2021-11-11T20:45:19.450 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:19 +0000 (0:00:00.049) 0:07:39.706 ***** 2021-11-11T20:45:19.478 INFO:teuthology.task.ansible.out: TASK [testnode : Touch /ceph-qa-ready] ***************************************** 2021-11-11T20:45:19.480 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:19.889 INFO:teuthology.task.ansible.out: TASK [testnode : List any leftover Ceph artifacts from previous jobs] ********** 2021-11-11T20:45:19.891 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=/var/run/) => {"ansible_loop_var": "item", "changed": false, "cmd": "find /var/run/ -name \"*ceph*\"", "delta": "0:00:01.006424", "end": "2021-11-11 20:45:19.855017", "failed_when_result": false, "item": "/var/run/", "rc": 0, "start": "2021-11-11 20:45:18.848593", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:20.078 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=/etc/systemd/system/) => {"ansible_loop_var": "item", "changed": false, "cmd": "find /etc/systemd/system/ -name \"*ceph*\"", "delta": "0:00:00.003018", "end": "2021-11-11 20:45:20.047391", "failed_when_result": false, "item": "/etc/systemd/system/", "rc": 0, "start": "2021-11-11 20:45:20.044373", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:20.289 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=/etc/ceph) => {"ansible_loop_var": "item", "changed": false, "cmd": "find /etc/ceph -name \"*ceph*\"", "delta": "0:00:00.002821", "end": "2021-11-11 20:45:20.249832", "failed_when_result": false, "item": "/etc/ceph", "msg": "non-zero return code", "rc": 1, "start": "2021-11-11 20:45:20.247011", "stderr": "find: ‘/etc/ceph’: No such file or directory", "stderr_lines": ["find: ‘/etc/ceph’: No such file or directory"], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:20.464 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=/var/log/) => {"ansible_loop_var": "item", "changed": false, "cmd": "find /var/log/ -name \"*ceph*\"", "delta": "0:00:00.003439", "end": "2021-11-11 20:45:20.432163", "failed_when_result": false, "item": "/var/log/", "rc": 0, "start": "2021-11-11 20:45:20.428724", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:45:20.487 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:20 +0000 (0:00:01.037) 0:07:40.743 ***** 2021-11-11T20:45:20.514 INFO:teuthology.task.ansible.out: TASK [testnode : Delete any leftover Ceph artifacts from previous jobs] ******** 2021-11-11T20:45:20.537 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:20 +0000 (0:00:00.049) 0:07:40.792 ***** 2021-11-11T20:45:20.565 INFO:teuthology.task.ansible.out: TASK [testnode : Touch /ceph-qa-ready] ***************************************** 2021-11-11T20:45:20.566 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:20.642 INFO:teuthology.task.ansible.out: PLAY [testnodes,senta,vossi,jenkins_slaves] ************************************ 2021-11-11T20:45:20.663 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:20 +0000 (0:00:00.126) 0:07:40.919 ***** 2021-11-11T20:45:20.685 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:20 +0000 (0:00:00.021) 0:07:40.940 ***** 2021-11-11T20:45:21.325 INFO:teuthology.task.ansible.out: TASK [Gathering Facts] ********************************************************* ok: [smithi096.front.sepia.ceph.com] 2021-11-11T20:45:21.358 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] 2021-11-11T20:45:21.385 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:21 +0000 (0:00:00.700) 0:07:41.641 ***** 2021-11-11T20:45:21.409 INFO:teuthology.task.ansible.out: TASK [container-host : set_fact] *********************************************** 2021-11-11T20:45:21.409 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:21.431 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:21 +0000 (0:00:00.046) 0:07:41.687 ***** 2021-11-11T20:45:21.453 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:21 +0000 (0:00:00.021) 0:07:41.709 ***** 2021-11-11T20:45:21.473 INFO:teuthology.task.ansible.out: TASK [container-host : set_fact] *********************************************** 2021-11-11T20:45:21.475 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"package_manager": "yum"}, "changed": false} 2021-11-11T20:45:21.496 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:21 +0000 (0:00:00.042) 0:07:41.752 ***** 2021-11-11T20:45:21.504 INFO:teuthology.task.ansible.out: TASK [container-host : set_fact] *********************************************** 2021-11-11T20:45:21.506 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:21.529 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:21 +0000 (0:00:00.032) 0:07:41.785 ***** 2021-11-11T20:45:21.537 INFO:teuthology.task.ansible.out: TASK [container-host : Including distro specific variables] ******************** 2021-11-11T20:45:21.539 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/container-host/vars/yum_systems.yml) => {"ansible_facts": {"container_packages": ["podman", "podman-docker"]}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/container-host/vars/yum_systems.yml"], "ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/container-host/vars/yum_systems.yml"} 2021-11-11T20:45:21.564 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:21 +0000 (0:00:00.034) 0:07:41.819 ***** 2021-11-11T20:45:21.572 INFO:teuthology.task.ansible.out: TASK [container-host : set_fact] *********************************************** ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"package_manager": "yum"}, "changed": false} 2021-11-11T20:45:21.598 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:21 +0000 (0:00:00.034) 0:07:41.854 ***** 2021-11-11T20:45:21.647 INFO:teuthology.task.ansible.out: TASK [container-host : Including distro specific variables] ******************** 2021-11-11T20:45:21.649 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/container-host/vars/yum_systems.yml) => {"ansible_facts": {"container_packages": ["podman", "podman-docker"]}, "ansible_included_var_files": ["/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/container-host/vars/yum_systems.yml"], "ansible_loop_var": "item", "changed": false, "item": "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/roles/container-host/vars/yum_systems.yml"} 2021-11-11T20:45:21.672 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:21 +0000 (0:00:00.073) 0:07:41.928 ***** 2021-11-11T20:45:43.128 INFO:teuthology.task.ansible.out: TASK [container-host : Install container packages] ***************************** changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "msg": "", "rc": 0, "results": ["Installed: podman-docker-3.2.3-0.11.module+el8.4.0+12050+ef972f71.noarch", "Installed: podman-catatonit-3.2.3-0.11.module+el8.4.0+12050+ef972f71.x86_64", "Installed: podman-3.2.3-0.11.module+el8.4.0+12050+ef972f71.x86_64", "Removed: podman-3.0.1-6.module+el8.4.0+10607+f4da7515.x86_64", "Removed: podman-catatonit-3.0.1-6.module+el8.4.0+10607+f4da7515.x86_64", "Removed: podman-docker-3.0.1-6.module+el8.4.0+10607+f4da7515.noarch"]} 2021-11-11T20:45:43.152 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:43 +0000 (0:00:21.479) 0:08:03.408 ***** 2021-11-11T20:45:43.193 INFO:teuthology.task.ansible.out: TASK [container-host : set_fact] *********************************************** 2021-11-11T20:45:43.194 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => {"ansible_facts": {"container_service_conf": "/etc/containers/registries.conf"}, "changed": false} 2021-11-11T20:45:43.216 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:43 +0000 (0:00:00.063) 0:08:03.471 ***** 2021-11-11T20:45:43.243 INFO:teuthology.task.ansible.out: TASK [container-host : set_fact] *********************************************** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:43.265 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:43 +0000 (0:00:00.050) 0:08:03.521 ***** 2021-11-11T20:45:43.497 INFO:teuthology.task.ansible.out: TASK [container-host : Create container_mirror_cert_paths] ********************* ok: [smithi096.front.sepia.ceph.com] => (item=/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000) => {"ansible_loop_var": "item", "changed": false, "gid": 0, "group": "root", "item": "/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000", "mode": "0755", "owner": "root", "path": "/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000", "secontext": "unconfined_u:object_r:etc_t:s0", "size": 4096, "state": "directory", "uid": 0} 2021-11-11T20:45:43.646 INFO:teuthology.task.ansible.out: TASK [container-host : Install container packages] ***************************** 2021-11-11T20:45:43.647 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "msg": "", "rc": 0, "results": ["Installed: podman-docker-3.2.3-0.11.module+el8.4.0+12050+ef972f71.noarch", "Installed: podman-catatonit-3.2.3-0.11.module+el8.4.0+12050+ef972f71.x86_64", "Installed: podman-3.2.3-0.11.module+el8.4.0+12050+ef972f71.x86_64", "Removed: podman-3.0.1-6.module+el8.4.0+10607+f4da7515.x86_64", "Removed: podman-catatonit-3.0.1-6.module+el8.4.0+10607+f4da7515.x86_64", "Removed: podman-docker-3.0.1-6.module+el8.4.0+10607+f4da7515.noarch"]} 2021-11-11T20:45:43.670 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:43 +0000 (0:00:00.404) 0:08:03.925 ***** 2021-11-11T20:45:43.676 INFO:teuthology.task.ansible.out: TASK [container-host : Create container_mirror_cert_paths] ********************* 2021-11-11T20:45:43.677 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] => (item=/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000) => {"ansible_loop_var": "item", "changed": false, "gid": 0, "group": "root", "item": "/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000", "mode": "0755", "owner": "root", "path": "/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000", "secontext": "unconfined_u:object_r:etc_t:s0", "size": 4096, "state": "directory", "uid": 0} 2021-11-11T20:45:43.712 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:43 +0000 (0:00:00.042) 0:08:03.968 ***** 2021-11-11T20:45:43.719 INFO:teuthology.task.ansible.out: TASK [container-host : set_fact] *********************************************** 2021-11-11T20:45:43.722 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"ansible_facts": {"container_service_conf": "/etc/containers/registries.conf"}, "changed": false} 2021-11-11T20:45:43.751 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:43 +0000 (0:00:00.036) 0:08:04.005 ***** 2021-11-11T20:45:43.781 INFO:teuthology.task.ansible.out: TASK [container-host : set_fact] *********************************************** skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:45:43.802 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:43 +0000 (0:00:00.052) 0:08:04.058 ***** 2021-11-11T20:45:44.034 INFO:teuthology.task.ansible.out: TASK [container-host : Create container_mirror_cert_paths] ********************* 2021-11-11T20:45:44.036 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000) => {"ansible_loop_var": "item", "changed": false, "gid": 0, "group": "root", "item": "/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000", "mode": "0755", "owner": "root", "path": "/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000", "secontext": "unconfined_u:object_r:etc_t:s0", "size": 4096, "state": "directory", "uid": 0} 2021-11-11T20:45:44.207 INFO:teuthology.task.ansible.out: TASK [container-host : Copy docker-mirror.front.sepia.ceph.com:5000 self-signed cert] *** ok: [smithi096.front.sepia.ceph.com] => (item=/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000) => {"ansible_loop_var": "item", "changed": false, "checksum": "bcdfbb1de91ef82e0e4ed8af725c9ad52593eb79", "dest": "/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000/docker-mirror.crt", "gid": 0, "group": "root", "item": "/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000", "mode": "0644", "owner": "root", "path": "/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000/docker-mirror.crt", "secontext": "system_u:object_r:cert_t:s0", "size": 2334, "state": "file", "uid": 0} 2021-11-11T20:45:44.216 INFO:teuthology.task.ansible.out: TASK [container-host : Create container_mirror_cert_paths] ********************* 2021-11-11T20:45:44.217 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000) => {"ansible_loop_var": "item", "changed": false, "gid": 0, "group": "root", "item": "/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000", "mode": "0755", "owner": "root", "path": "/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000", "secontext": "unconfined_u:object_r:etc_t:s0", "size": 4096, "state": "directory", "uid": 0} 2021-11-11T20:45:44.241 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:44 +0000 (0:00:00.438) 0:08:04.497 ***** 2021-11-11T20:45:44.642 INFO:teuthology.task.ansible.out: TASK [container-host : Copy docker-mirror.front.sepia.ceph.com:5000 self-signed cert] *** ok: [smithi096.front.sepia.ceph.com] => (item=/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000) => {"ansible_loop_var": "item", "changed": false, "checksum": "bcdfbb1de91ef82e0e4ed8af725c9ad52593eb79", "dest": "/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000/docker-mirror.crt", "gid": 0, "group": "root", "item": "/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000", "mode": "0644", "owner": "root", "path": "/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000/docker-mirror.crt", "secontext": "system_u:object_r:etc_t:s0", "size": 2334, "state": "file", "uid": 0} 2021-11-11T20:45:44.666 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:44 +0000 (0:00:00.424) 0:08:04.921 ***** 2021-11-11T20:45:44.737 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000) => {"ansible_loop_var": "item", "changed": false, "checksum": "bcdfbb1de91ef82e0e4ed8af725c9ad52593eb79", "dest": "/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000/docker-mirror.crt", "gid": 0, "group": "root", "item": "/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000", "mode": "0644", "owner": "root", "path": "/etc/docker/certs.d/docker-mirror.front.sepia.ceph.com:5000/docker-mirror.crt", "secontext": "system_u:object_r:cert_t:s0", "size": 2334, "state": "file", "uid": 0} 2021-11-11T20:45:45.177 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => (item=/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000) => {"ansible_loop_var": "item", "changed": false, "checksum": "bcdfbb1de91ef82e0e4ed8af725c9ad52593eb79", "dest": "/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000/docker-mirror.crt", "gid": 0, "group": "root", "item": "/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000", "mode": "0644", "owner": "root", "path": "/etc/containers/certs.d/docker-mirror.front.sepia.ceph.com:5000/docker-mirror.crt", "secontext": "system_u:object_r:etc_t:s0", "size": 2334, "state": "file", "uid": 0} 2021-11-11T20:45:45.203 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:45 +0000 (0:00:00.536) 0:08:05.458 ***** 2021-11-11T20:45:58.766 INFO:teuthology.task.ansible.out: TASK [container-host : Ensure git is installed] ******************************** ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:45:58.789 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:58 +0000 (0:00:13.587) 0:08:19.045 ***** 2021-11-11T20:45:59.242 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "msg": "Nothing to do", "rc": 0, "results": []} 2021-11-11T20:45:59.281 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:45:59 +0000 (0:00:00.491) 0:08:19.537 ***** 2021-11-11T20:46:01.345 INFO:teuthology.task.ansible.out: TASK [container-host : Install registries-conf-ctl] **************************** ok: [smithi096.front.sepia.ceph.com] => {"changed": false, "cmd": ["/bin/pip3", "install", "-U", "git+https://github.com/sebastian-philipp/registries-conf-ctl"], "name": ["git+https://github.com/sebastian-philipp/registries-conf-ctl"], "requirements": null, "state": "latest", "stderr": "WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead.\n", "stderr_lines": ["WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead."], "stdout": "Collecting git+https://github.com/sebastian-philipp/registries-conf-ctl\n Cloning https://github.com/sebastian-philipp/registries-conf-ctl to ./pip-h3csmsqk-build\nRequirement already up-to-date: toml in /usr/local/lib/python3.6/site-packages (from registries-conf-ctl==1.0)\nRequirement already up-to-date: docopt in /usr/local/lib/python3.6/site-packages (from registries-conf-ctl==1.0)\nInstalling collected packages: registries-conf-ctl\n Found existing installation: registries-conf-ctl 1.0\n Uninstalling registries-conf-ctl-1.0:\n Successfully uninstalled registries-conf-ctl-1.0\n Running setup.py install for registries-conf-ctl: started\n Running setup.py install for registries-conf-ctl: finished with status 'done'\nSuccessfully installed registries-conf-ctl-1.0\n", "stdout_lines": ["Collecting git+https://github.com/sebastian-philipp/registries-conf-ctl", " Cloning https://github.com/sebastian-philipp/registries-conf-ctl to ./pip-h3csmsqk-build", "Requirement already up-to-date: toml in /usr/local/lib/python3.6/site-packages (from registries-conf-ctl==1.0)", "Requirement already up-to-date: docopt in /usr/local/lib/python3.6/site-packages (from registries-conf-ctl==1.0)", "Installing collected packages: registries-conf-ctl", " Found existing installation: registries-conf-ctl 1.0", " Uninstalling registries-conf-ctl-1.0 2021-11-11T20:46:01.346 INFO:teuthology.task.ansible.out::", " Successfully uninstalled registries-conf-ctl-1.0", " Running setup.py install for registries-conf-ctl: started", " Running setup.py install for registries-conf-ctl: finished with status 'done'", "Successfully installed registries-conf-ctl-1.0"], "version": null, "virtualenv": null} 2021-11-11T20:46:01.369 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:46:01 +0000 (0:00:02.088) 0:08:21.625 ***** 2021-11-11T20:46:01.400 INFO:teuthology.task.ansible.out: TASK [container-host : Check for docker's daemon.json] ************************* skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:46:01.426 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:46:01 +0000 (0:00:00.056) 0:08:21.681 ***** 2021-11-11T20:46:01.457 INFO:teuthology.task.ansible.out: TASK [container-host : Create /etc/containers/registries.conf if necessary] **** skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:46:01.478 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:46:01 +0000 (0:00:00.052) 0:08:21.734 ***** 2021-11-11T20:46:01.856 INFO:teuthology.task.ansible.out: TASK [container-host : Add local docker.io registry mirror] ******************** 2021-11-11T20:46:01.859 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "cmd": ["registries-conf-ctl", "add-mirror", "docker.io", "docker-mirror.front.sepia.ceph.com:5000"], "delta": "0:00:00.164868", "end": "2021-11-11 20:46:01.829683", "rc": 0, "start": "2021-11-11 20:46:01.664815", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} TASK [container-host : Install registries-conf-ctl] **************************** ok: [smithi198.front.sepia.ceph.com] => {"changed": false, "cmd": ["/bin/pip3", "install", "-U", "git+https://github.com/sebastian-philipp/registries-conf-ctl"], "name": ["git+https://github.com/sebastian-philipp/registries-conf-ctl"], "requirements": null, "state": "latest", "stderr": "WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead.\n", "stderr_lines": ["WARNING: Running pip install with root privileges is generally not a good idea. Try `pip3 install --user` instead."], "stdout": "Collecting git+https://github.com/sebastian-philipp/registries-conf-ctl\n Cloning https://github.com/sebastian-philipp/registries-conf-ctl to ./pip-gw3fehah-build\nRequirement already up-to-date: toml in /usr/local/lib/python3.6/site-packages (from registries-conf-ctl==1.0)\nRequirement already up-to-date: docopt in /usr/local/lib/python3.6/site-packages (from registries-conf-ctl==1.0)\nInstalling collected packages: registries-conf-ctl\n Found existing installation: registries-conf-ctl 1.0\n Uninstalling registries-conf-ctl-1.0:\n Successfully uninstalled registries-conf-ctl-1.0\n Running setup.py install for registries-conf-ctl: started\n Running setup.py install for registries-conf-ctl: finished with status 'done'\nSuccessfully installed registries-conf-ctl-1.0\n", "stdout_lines": ["Collecting git+https://github.com/sebastian-philipp/registries-conf-ctl", " Cloning https://github.com/sebastian-philipp/registries-conf-ctl to ./pip-gw3fehah-build", "Requirement already up-to-date: toml 2021-11-11T20:46:01.860 INFO:teuthology.task.ansible.out: in /usr/local/lib/python3.6/site-packages (from registries-conf-ctl==1.0)", "Requirement already up-to-date: docopt in /usr/local/lib/python3.6/site-packages (from registries-conf-ctl==1.0)", "Installing collected packages: registries-conf-ctl", " Found existing installation: registries-conf-ctl 1.0", " Uninstalling registries-conf-ctl-1.0:", " Successfully uninstalled registries-conf-ctl-1.0", " Running setup.py install for registries-conf-ctl: started", " Running setup.py install for registries-conf-ctl: finished with status 'done'", "Successfully installed registries-conf-ctl-1.0"], "version": null, "virtualenv": null} 2021-11-11T20:46:01.883 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:46:01 +0000 (0:00:00.405) 0:08:22.139 ***** 2021-11-11T20:46:01.904 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:46:01 +0000 (0:00:00.021) 0:08:22.160 ***** 2021-11-11T20:46:01.922 INFO:teuthology.task.ansible.out: TASK [container-host : Restart docker service] ********************************* 2021-11-11T20:46:01.924 INFO:teuthology.task.ansible.out:skipping: [smithi096.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:46:01.947 INFO:teuthology.task.ansible.out: TASK [container-host : Check for docker's daemon.json] ************************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:46:01.981 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:46:01 +0000 (0:00:00.075) 0:08:22.235 ***** 2021-11-11T20:46:02.009 INFO:teuthology.task.ansible.out: TASK [container-host : Create /etc/containers/registries.conf if necessary] **** 2021-11-11T20:46:02.010 INFO:teuthology.task.ansible.out:skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:46:02.040 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:46:02 +0000 (0:00:00.060) 0:08:22.296 ***** 2021-11-11T20:46:02.400 INFO:teuthology.task.ansible.out: TASK [container-host : Add local docker.io registry mirror] ******************** 2021-11-11T20:46:02.402 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "cmd": ["registries-conf-ctl", "add-mirror", "docker.io", "docker-mirror.front.sepia.ceph.com:5000"], "delta": "0:00:00.154303", "end": "2021-11-11 20:46:02.376537", "rc": 0, "start": "2021-11-11 20:46:02.222234", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} 2021-11-11T20:46:02.423 INFO:teuthology.task.ansible.out:Thursday 11 November 2021 20:46:02 +0000 (0:00:00.382) 0:08:22.679 ***** 2021-11-11T20:46:02.454 INFO:teuthology.task.ansible.out: TASK [container-host : Restart docker service] ********************************* skipping: [smithi198.front.sepia.ceph.com] => {"changed": false, "skip_reason": "Conditional result was False"} 2021-11-11T20:46:02.526 INFO:teuthology.task.ansible.out: PLAY [cobbler] ***************************************************************** 2021-11-11T20:46:02.527 INFO:teuthology.task.ansible.out:skipping: no hosts matched PLAY [paddles] ***************************************************************** skipping: no hosts matched 2021-11-11T20:46:02.529 INFO:teuthology.task.ansible.out: PLAY [pulpito] ***************************************************************** skipping: no hosts matched 2021-11-11T20:46:02.531 INFO:teuthology.task.ansible.out: PLAY [testnodes] *************************************************************** 2021-11-11T20:46:02.553 INFO:teuthology.task.ansible.out: TASK [Gathering Facts] ********************************************************* Thursday 11 November 2021 20:46:02 +0000 (0:00:00.123) 0:08:22.803 ***** 2021-11-11T20:46:03.213 INFO:teuthology.task.ansible.out:ok: [smithi096.front.sepia.ceph.com] 2021-11-11T20:46:04.245 INFO:teuthology.task.ansible.out:ok: [smithi198.front.sepia.ceph.com] 2021-11-11T20:46:04.272 INFO:teuthology.task.ansible.out: TASK [Touch /ceph-qa-ready] **************************************************** Thursday 11 November 2021 20:46:04 +0000 (0:00:01.724) 0:08:24.527 ***** 2021-11-11T20:46:04.489 INFO:teuthology.task.ansible.out:changed: [smithi096.front.sepia.ceph.com] => {"changed": true, "dest": "/ceph-qa-ready", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "secontext": "unconfined_u:object_r:etc_runtime_t:s0", "size": 0, "state": "file", "uid": 0} 2021-11-11T20:46:04.501 INFO:teuthology.task.ansible.out:changed: [smithi198.front.sepia.ceph.com] => {"changed": true, "dest": "/ceph-qa-ready", "gid": 0, "group": "root", "mode": "0644", "owner": "root", "secontext": "unconfined_u:object_r:etc_runtime_t:s0", "size": 0, "state": "file", "uid": 0} 2021-11-11T20:46:04.504 INFO:teuthology.task.ansible.out: PLAY RECAP ********************************************************************* smithi096.front.sepia.ceph.com : ok=145 changed=45 unreachable=0 failed=0 skipped=174 rescued=0 ignored=4 smithi198.front.sepia.ceph.com : ok=145 changed=45 unreachable=0 failed=0 skipped=174 rescued=0 ignored=4 Thursday 11 November 2021 20:46:04 +0000 (0:00:00.232) 0:08:24.759 ***** =============================================================================== 2021-11-11T20:46:04.506 INFO:teuthology.task.ansible.out:Create all admin users with sudo access. ------------------------------- 59.63s common : nrpe - Ensure SELinux policy is up to date -------------------- 52.89s users : Update authorized_keys using the keys repo --------------------- 35.55s testnode : Ensure ceph packages are not present. ----------------------- 34.99s testnode : Install packages -------------------------------------------- 33.32s testnode : Ensure ceph dependency packages are not present. ------------ 29.09s container-host : Install container packages ---------------------------- 21.48s common : Register with subscription-manager. --------------------------- 18.99s Remove revoked users --------------------------------------------------- 18.80s testnode : Install GPG ------------------------------------------------- 15.15s common : nrpe - Load SELinux policy package ---------------------------- 14.63s common : nrpe - Remove SELinux policy package -------------------------- 14.60s container-host : Ensure git is installed ------------------------------- 13.59s common : Install Kerberos Packages (RedHat) ---------------------------- 13.47s testnode : Install epel packages --------------------------------------- 13.46s testnode : Zap all non-root disks --------------------------------------- 7.39s testnode : Make sure rpm dependencies are installed --------------------- 2.74s testnode : Remove packages ---------------------------------------------- 2.45s Gathering Facts --------------------------------------------------------- 2.42s testnode : Install GPG keys --------------------------------------------- 2.41s 2021-11-11T20:46:04.912 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi096.front.sepia.ceph.com'), Remote(name='ubuntu@smithi198.front.sepia.ceph.com')] 2021-11-11T20:46:04.913 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi096.front.sepia.ceph.com' 2021-11-11T20:46:04.915 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi096.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2021-11-11T20:46:04.983 DEBUG:teuthology.orchestra.run.smithi096:> true 2021-11-11T20:46:05.049 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi198.front.sepia.ceph.com' 2021-11-11T20:46:05.050 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi198.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2021-11-11T20:46:05.122 DEBUG:teuthology.orchestra.run.smithi198:> true 2021-11-11T20:46:05.187 INFO:teuthology.run_tasks:Running task clock... 2021-11-11T20:46:05.200 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2021-11-11T20:46:05.201 INFO:teuthology.orchestra.run:Running command with timeout 360 2021-11-11T20:46:05.203 DEBUG:teuthology.orchestra.run.smithi096:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2021-11-11T20:46:05.232 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2021-11-11T20:46:05.246 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2021-11-11T20:46:05.277 INFO:teuthology.orchestra.run.smithi096.stderr:sudo: ntpd: command not found 2021-11-11T20:46:05.287 INFO:teuthology.orchestra.run.smithi096.stdout:506 Cannot talk to daemon 2021-11-11T20:46:05.301 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to start ntp.service: Unit ntp.service not found. 2021-11-11T20:46:05.315 INFO:teuthology.orchestra.run.smithi096.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2021-11-11T20:46:05.372 INFO:teuthology.orchestra.run.smithi096.stderr:bash: ntpq: command not found 2021-11-11T20:46:05.376 INFO:teuthology.orchestra.run.smithi096.stdout:210 Number of sources = 4 2021-11-11T20:46:05.376 INFO:teuthology.orchestra.run.smithi096.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2021-11-11T20:46:05.376 INFO:teuthology.orchestra.run.smithi096.stdout:=============================================================================== 2021-11-11T20:46:05.376 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2021-11-11T20:46:05.377 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2021-11-11T20:46:05.378 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2021-11-11T20:46:05.378 INFO:teuthology.orchestra.run.smithi096.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2021-11-11T20:46:05.379 INFO:teuthology.orchestra.run:Running command with timeout 360 2021-11-11T20:46:05.379 DEBUG:teuthology.orchestra.run.smithi198:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2021-11-11T20:46:05.410 INFO:teuthology.orchestra.run.smithi198.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2021-11-11T20:46:05.425 INFO:teuthology.orchestra.run.smithi198.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2021-11-11T20:46:05.460 INFO:teuthology.orchestra.run.smithi198.stderr:sudo: ntpd: command not found 2021-11-11T20:46:05.470 INFO:teuthology.orchestra.run.smithi198.stdout:506 Cannot talk to daemon 2021-11-11T20:46:05.485 INFO:teuthology.orchestra.run.smithi198.stderr:Failed to start ntp.service: Unit ntp.service not found. 2021-11-11T20:46:05.499 INFO:teuthology.orchestra.run.smithi198.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2021-11-11T20:46:05.560 INFO:teuthology.orchestra.run.smithi198.stderr:bash: ntpq: command not found 2021-11-11T20:46:05.563 INFO:teuthology.orchestra.run.smithi198.stdout:210 Number of sources = 4 2021-11-11T20:46:05.563 INFO:teuthology.orchestra.run.smithi198.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2021-11-11T20:46:05.563 INFO:teuthology.orchestra.run.smithi198.stdout:=============================================================================== 2021-11-11T20:46:05.563 INFO:teuthology.orchestra.run.smithi198.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2021-11-11T20:46:05.564 INFO:teuthology.orchestra.run.smithi198.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2021-11-11T20:46:05.564 INFO:teuthology.orchestra.run.smithi198.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2021-11-11T20:46:05.564 INFO:teuthology.orchestra.run.smithi198.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2021-11-11T20:46:05.564 INFO:teuthology.run_tasks:Running task install... 2021-11-11T20:46:05.575 DEBUG:teuthology.task.install:project ceph 2021-11-11T20:46:05.576 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098'}} 2021-11-11T20:46:05.576 DEBUG:teuthology.task.install:config {'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098'} 2021-11-11T20:46:05.579 INFO:teuthology.task.install:Using flavor: default 2021-11-11T20:46:05.586 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2021-11-11T20:46:05.587 INFO:teuthology.task.install:extra packages: [] 2021-11-11T20:46:05.587 DEBUG:teuthology.orchestra.run.smithi096:> sudo lsb_release -is 2021-11-11T20:46:05.626 INFO:teuthology.orchestra.run.smithi096.stdout:RedHatEnterprise 2021-11-11T20:46:05.627 DEBUG:teuthology.misc:System to be installed: RedHatEnterprise 2021-11-11T20:46:05.628 DEBUG:teuthology.orchestra.run.smithi198:> sudo lsb_release -is 2021-11-11T20:46:05.629 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098', 'tag': None, 'wait_for_package': False} 2021-11-11T20:46:05.632 DEBUG:teuthology.orchestra.run.smithi096:> sudo lsb_release -is 2021-11-11T20:46:05.672 INFO:teuthology.orchestra.run.smithi198.stdout:RedHatEnterprise 2021-11-11T20:46:05.673 DEBUG:teuthology.misc:System to be installed: RedHatEnterprise 2021-11-11T20:46:05.674 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098', 'tag': None, 'wait_for_package': False} 2021-11-11T20:46:05.674 DEBUG:teuthology.orchestra.run.smithi198:> sudo lsb_release -is 2021-11-11T20:46:05.705 INFO:teuthology.orchestra.run.smithi096.stdout:RedHatEnterprise 2021-11-11T20:46:05.707 DEBUG:teuthology.misc:System to be installed: RedHatEnterprise 2021-11-11T20:46:05.708 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=a3a8018ba3cfe536252b397411bf18b7828c6098 2021-11-11T20:46:05.759 INFO:teuthology.orchestra.run.smithi198.stdout:RedHatEnterprise 2021-11-11T20:46:05.760 DEBUG:teuthology.misc:System to be installed: RedHatEnterprise 2021-11-11T20:46:05.760 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=a3a8018ba3cfe536252b397411bf18b7828c6098 2021-11-11T20:46:05.923 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/cloudtiering/a3a8018ba3cfe536252b397411bf18b7828c6098/centos/8/flavors/default/ 2021-11-11T20:46:05.924 INFO:teuthology.task.install.rpm:Package version is 17.0.0-8977.ga3a8018b 2021-11-11T20:46:05.972 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/cloudtiering/a3a8018ba3cfe536252b397411bf18b7828c6098/centos/8/flavors/default/ 2021-11-11T20:46:05.973 INFO:teuthology.task.install.rpm:Package version is 17.0.0-8977.ga3a8018b 2021-11-11T20:46:06.149 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/cloudtiering/a3a8018ba3cfe536252b397411bf18b7828c6098/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/cloudtiering/a3a8018ba3cfe536252b397411bf18b7828c6098/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/cloudtiering/a3a8018ba3cfe536252b397411bf18b7828c6098/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2021-11-11T20:46:06.150 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:46:06.150 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd of=/etc/yum.repos.d/ceph.repo 2021-11-11T20:46:06.175 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2021-11-11T20:46:06.176 DEBUG:teuthology.orchestra.run.smithi198:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/a3a8018ba3cfe536252b397411bf18b7828c6098/;g' /etc/yum.repos.d/ceph.repo ; fi 2021-11-11T20:46:06.190 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/cloudtiering/a3a8018ba3cfe536252b397411bf18b7828c6098/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/cloudtiering/a3a8018ba3cfe536252b397411bf18b7828c6098/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/cloudtiering/a3a8018ba3cfe536252b397411bf18b7828c6098/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2021-11-11T20:46:06.191 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:46:06.191 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/yum.repos.d/ceph.repo 2021-11-11T20:46:06.215 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2021-11-11T20:46:06.216 DEBUG:teuthology.orchestra.run.smithi096:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/a3a8018ba3cfe536252b397411bf18b7828c6098/;g' /etc/yum.repos.d/ceph.repo ; fi 2021-11-11T20:46:06.241 DEBUG:teuthology.orchestra.run.smithi198:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2021-11-11T20:46:06.281 DEBUG:teuthology.orchestra.run.smithi096:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2021-11-11T20:46:06.315 DEBUG:teuthology.orchestra.run.smithi198:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2021-11-11T20:46:06.356 DEBUG:teuthology.orchestra.run.smithi096:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2021-11-11T20:46:06.380 INFO:teuthology.orchestra.run.smithi198.stdout:check_obsoletes = 1 2021-11-11T20:46:06.381 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum clean all 2021-11-11T20:46:06.419 INFO:teuthology.orchestra.run.smithi096.stdout:check_obsoletes = 1 2021-11-11T20:46:06.421 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum clean all 2021-11-11T20:46:06.734 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:46:06.775 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:46:07.042 INFO:teuthology.orchestra.run.smithi198.stdout:65 files removed 2021-11-11T20:46:07.085 INFO:teuthology.orchestra.run.smithi096.stdout:65 files removed 2021-11-11T20:46:07.088 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph-radosgw 2021-11-11T20:46:07.133 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-radosgw 2021-11-11T20:46:07.402 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:46:07.447 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:46:08.235 INFO:teuthology.orchestra.run.smithi096.stdout:ceph packages for x86_64 183 kB/s | 75 kB 00:00 2021-11-11T20:46:08.246 INFO:teuthology.orchestra.run.smithi198.stdout:ceph packages for x86_64 167 kB/s | 75 kB 00:00 2021-11-11T20:46:08.594 INFO:teuthology.orchestra.run.smithi096.stdout:ceph noarch packages 27 kB/s | 9.1 kB 00:00 2021-11-11T20:46:08.595 INFO:teuthology.orchestra.run.smithi198.stdout:ceph noarch packages 28 kB/s | 9.1 kB 00:00 2021-11-11T20:46:08.921 INFO:teuthology.orchestra.run.smithi198.stdout:ceph source packages 4.6 kB/s | 1.4 kB 00:00 2021-11-11T20:46:08.923 INFO:teuthology.orchestra.run.smithi096.stdout:ceph source packages 4.6 kB/s | 1.4 kB 00:00 2021-11-11T20:46:09.023 INFO:teuthology.orchestra.run.smithi198.stdout:Copr repo for python3-asyncssh owned by ceph 42 kB/s | 3.5 kB 00:00 2021-11-11T20:46:09.028 INFO:teuthology.orchestra.run.smithi096.stdout:Copr repo for python3-asyncssh owned by ceph 40 kB/s | 3.5 kB 00:00 2021-11-11T20:46:10.084 INFO:teuthology.orchestra.run.smithi198.stdout:Apache Arrow 149 kB/s | 156 kB 00:01 2021-11-11T20:46:10.092 INFO:teuthology.orchestra.run.smithi096.stdout:Apache Arrow 149 kB/s | 156 kB 00:01 2021-11-11T20:46:10.784 INFO:teuthology.orchestra.run.smithi096.stdout:Extra Packages for Enterprise Linux 16 MB/s | 11 MB 00:00 2021-11-11T20:46:10.851 INFO:teuthology.orchestra.run.smithi198.stdout:Extra Packages for Enterprise Linux 15 MB/s | 11 MB 00:00 2021-11-11T20:46:12.915 INFO:teuthology.orchestra.run.smithi096.stdout:lab-extras 4.2 MB/s | 24 kB 00:00 2021-11-11T20:46:12.963 INFO:teuthology.orchestra.run.smithi198.stdout:lab-extras 4.4 MB/s | 24 kB 00:00 2021-11-11T20:46:13.226 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 129 MB/s | 33 MB 00:00 2021-11-11T20:46:13.288 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 142 MB/s | 38 MB 00:00 2021-11-11T20:46:18.429 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 142 MB/s | 38 MB 00:00 2021-11-11T20:46:18.712 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 130 MB/s | 33 MB 00:00 2021-11-11T20:46:27.035 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:46:27.038 INFO:teuthology.orchestra.run.smithi096.stdout:==================================================================================================== 2021-11-11T20:46:27.038 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2021-11-11T20:46:27.038 INFO:teuthology.orchestra.run.smithi096.stdout:==================================================================================================== 2021-11-11T20:46:27.038 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:46:27.039 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-radosgw x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 9.1 M 2021-11-11T20:46:27.039 INFO:teuthology.orchestra.run.smithi096.stdout:Upgrading: 2021-11-11T20:46:27.039 INFO:teuthology.orchestra.run.smithi096.stdout: librados2 x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 3.4 M 2021-11-11T20:46:27.039 INFO:teuthology.orchestra.run.smithi096.stdout: librbd1 x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 3.9 M 2021-11-11T20:46:27.039 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2021-11-11T20:46:27.040 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-base x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 5.2 M 2021-11-11T20:46:27.040 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-common x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 20 M 2021-11-11T20:46:27.040 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-selinux x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 24 k 2021-11-11T20:46:27.040 INFO:teuthology.orchestra.run.smithi096.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2021-11-11T20:46:27.041 INFO:teuthology.orchestra.run.smithi096.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2021-11-11T20:46:27.041 INFO:teuthology.orchestra.run.smithi096.stdout: libcephfs2 x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 786 k 2021-11-11T20:46:27.041 INFO:teuthology.orchestra.run.smithi096.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2021-11-11T20:46:27.041 INFO:teuthology.orchestra.run.smithi096.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2021-11-11T20:46:27.041 INFO:teuthology.orchestra.run.smithi096.stdout: libradosstriper1 x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 483 k 2021-11-11T20:46:27.042 INFO:teuthology.orchestra.run.smithi096.stdout: librdkafka x86_64 0.11.4-1.el8 rhel-8-for-x86_64-appstream-rpms 353 k 2021-11-11T20:46:27.042 INFO:teuthology.orchestra.run.smithi096.stdout: librgw2 x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 3.9 M 2021-11-11T20:46:27.042 INFO:teuthology.orchestra.run.smithi096.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2021-11-11T20:46:27.042 INFO:teuthology.orchestra.run.smithi096.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2021-11-11T20:46:27.043 INFO:teuthology.orchestra.run.smithi096.stdout: python3-ceph-argparse x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 45 k 2021-11-11T20:46:27.043 INFO:teuthology.orchestra.run.smithi096.stdout: python3-ceph-common x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 79 k 2021-11-11T20:46:27.043 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cephfs x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 178 k 2021-11-11T20:46:27.043 INFO:teuthology.orchestra.run.smithi096.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2021-11-11T20:46:27.044 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rados x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 366 k 2021-11-11T20:46:27.045 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rbd x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 346 k 2021-11-11T20:46:27.045 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rgw x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 109 k 2021-11-11T20:46:27.045 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:46:27.045 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:46:27.045 INFO:teuthology.orchestra.run.smithi096.stdout:==================================================================================================== 2021-11-11T20:46:27.046 INFO:teuthology.orchestra.run.smithi096.stdout:Install 21 Packages 2021-11-11T20:46:27.046 INFO:teuthology.orchestra.run.smithi096.stdout:Upgrade 2 Packages 2021-11-11T20:46:27.046 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:46:27.047 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 49 M 2021-11-11T20:46:27.047 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:46:27.196 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:46:27.198 INFO:teuthology.orchestra.run.smithi198.stdout:==================================================================================================== 2021-11-11T20:46:27.198 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repository Size 2021-11-11T20:46:27.199 INFO:teuthology.orchestra.run.smithi198.stdout:==================================================================================================== 2021-11-11T20:46:27.199 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:46:27.199 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-radosgw x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 9.1 M 2021-11-11T20:46:27.199 INFO:teuthology.orchestra.run.smithi198.stdout:Upgrading: 2021-11-11T20:46:27.200 INFO:teuthology.orchestra.run.smithi198.stdout: librados2 x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 3.4 M 2021-11-11T20:46:27.200 INFO:teuthology.orchestra.run.smithi198.stdout: librbd1 x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 3.9 M 2021-11-11T20:46:27.200 INFO:teuthology.orchestra.run.smithi198.stdout:Installing dependencies: 2021-11-11T20:46:27.200 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-base x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 5.2 M 2021-11-11T20:46:27.201 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-common x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 20 M 2021-11-11T20:46:27.201 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-selinux x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 24 k 2021-11-11T20:46:27.201 INFO:teuthology.orchestra.run.smithi198.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2021-11-11T20:46:27.201 INFO:teuthology.orchestra.run.smithi198.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2021-11-11T20:46:27.201 INFO:teuthology.orchestra.run.smithi198.stdout: libcephfs2 x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 786 k 2021-11-11T20:46:27.202 INFO:teuthology.orchestra.run.smithi198.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2021-11-11T20:46:27.202 INFO:teuthology.orchestra.run.smithi198.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2021-11-11T20:46:27.202 INFO:teuthology.orchestra.run.smithi198.stdout: libradosstriper1 x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 483 k 2021-11-11T20:46:27.202 INFO:teuthology.orchestra.run.smithi198.stdout: librdkafka x86_64 0.11.4-1.el8 rhel-8-for-x86_64-appstream-rpms 353 k 2021-11-11T20:46:27.203 INFO:teuthology.orchestra.run.smithi198.stdout: librgw2 x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 3.9 M 2021-11-11T20:46:27.203 INFO:teuthology.orchestra.run.smithi198.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2021-11-11T20:46:27.203 INFO:teuthology.orchestra.run.smithi198.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2021-11-11T20:46:27.203 INFO:teuthology.orchestra.run.smithi198.stdout: python3-ceph-argparse x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 45 k 2021-11-11T20:46:27.203 INFO:teuthology.orchestra.run.smithi198.stdout: python3-ceph-common x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 79 k 2021-11-11T20:46:27.204 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cephfs x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 178 k 2021-11-11T20:46:27.204 INFO:teuthology.orchestra.run.smithi198.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2021-11-11T20:46:27.204 INFO:teuthology.orchestra.run.smithi198.stdout: python3-rados x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 366 k 2021-11-11T20:46:27.204 INFO:teuthology.orchestra.run.smithi198.stdout: python3-rbd x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 346 k 2021-11-11T20:46:27.205 INFO:teuthology.orchestra.run.smithi198.stdout: python3-rgw x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 109 k 2021-11-11T20:46:27.205 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:46:27.205 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:46:27.205 INFO:teuthology.orchestra.run.smithi198.stdout:==================================================================================================== 2021-11-11T20:46:27.205 INFO:teuthology.orchestra.run.smithi198.stdout:Install 21 Packages 2021-11-11T20:46:27.206 INFO:teuthology.orchestra.run.smithi198.stdout:Upgrade 2 Packages 2021-11-11T20:46:27.206 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:46:27.207 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 49 M 2021-11-11T20:46:27.207 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:46:27.945 INFO:teuthology.orchestra.run.smithi096.stdout:(1/23): ceph-base-17.0.0-8977.ga3a8018b.el8.x86 5.8 MB/s | 5.2 MB 00:00 2021-11-11T20:46:28.015 INFO:teuthology.orchestra.run.smithi096.stdout:(2/23): ceph-selinux-17.0.0-8977.ga3a8018b.el8. 346 kB/s | 24 kB 00:00 2021-11-11T20:46:28.176 INFO:teuthology.orchestra.run.smithi096.stdout:(3/23): libcephfs2-17.0.0-8977.ga3a8018b.el8.x8 4.8 MB/s | 786 kB 00:00 2021-11-11T20:46:28.208 INFO:teuthology.orchestra.run.smithi198.stdout:(1/23): ceph-base-17.0.0-8977.ga3a8018b.el8.x86 5.2 MB/s | 5.2 MB 00:01 2021-11-11T20:46:28.259 INFO:teuthology.orchestra.run.smithi198.stdout:(2/23): ceph-selinux-17.0.0-8977.ga3a8018b.el8. 466 kB/s | 24 kB 00:00 2021-11-11T20:46:28.353 INFO:teuthology.orchestra.run.smithi096.stdout:(4/23): libradosstriper1-17.0.0-8977.ga3a8018b. 2.7 MB/s | 483 kB 00:00 2021-11-11T20:46:28.397 INFO:teuthology.orchestra.run.smithi198.stdout:(3/23): libcephfs2-17.0.0-8977.ga3a8018b.el8.x8 5.6 MB/s | 786 kB 00:00 2021-11-11T20:46:28.612 INFO:teuthology.orchestra.run.smithi198.stdout:(4/23): libradosstriper1-17.0.0-8977.ga3a8018b. 2.2 MB/s | 483 kB 00:00 2021-11-11T20:46:28.779 INFO:teuthology.orchestra.run.smithi096.stdout:(5/23): librgw2-17.0.0-8977.ga3a8018b.el8.x86_6 9.1 MB/s | 3.9 MB 00:00 2021-11-11T20:46:28.843 INFO:teuthology.orchestra.run.smithi096.stdout:(6/23): python3-ceph-argparse-17.0.0-8977.ga3a8 700 kB/s | 45 kB 00:00 2021-11-11T20:46:28.903 INFO:teuthology.orchestra.run.smithi096.stdout:(7/23): python3-ceph-common-17.0.0-8977.ga3a801 1.3 MB/s | 79 kB 00:00 2021-11-11T20:46:29.032 INFO:teuthology.orchestra.run.smithi096.stdout:(8/23): python3-cephfs-17.0.0-8977.ga3a8018b.el 1.3 MB/s | 178 kB 00:00 2021-11-11T20:46:29.268 INFO:teuthology.orchestra.run.smithi096.stdout:(9/23): python3-rados-17.0.0-8977.ga3a8018b.el8 1.5 MB/s | 366 kB 00:00 2021-11-11T20:46:29.347 INFO:teuthology.orchestra.run.smithi198.stdout:(5/23): librgw2-17.0.0-8977.ga3a8018b.el8.x86_6 5.3 MB/s | 3.9 MB 00:00 2021-11-11T20:46:29.361 INFO:teuthology.orchestra.run.smithi096.stdout:(10/23): python3-rbd-17.0.0-8977.ga3a8018b.el8. 3.6 MB/s | 346 kB 00:00 2021-11-11T20:46:29.405 INFO:teuthology.orchestra.run.smithi198.stdout:(6/23): python3-ceph-argparse-17.0.0-8977.ga3a8 781 kB/s | 45 kB 00:00 2021-11-11T20:46:29.414 INFO:teuthology.orchestra.run.smithi096.stdout:(11/23): python3-rgw-17.0.0-8977.ga3a8018b.el8. 2.0 MB/s | 109 kB 00:00 2021-11-11T20:46:29.458 INFO:teuthology.orchestra.run.smithi198.stdout:(7/23): python3-ceph-common-17.0.0-8977.ga3a801 1.5 MB/s | 79 kB 00:00 2021-11-11T20:46:29.515 INFO:teuthology.orchestra.run.smithi198.stdout:(8/23): python3-cephfs-17.0.0-8977.ga3a8018b.el 3.2 MB/s | 178 kB 00:00 2021-11-11T20:46:29.612 INFO:teuthology.orchestra.run.smithi096.stdout:(12/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.5 MB/s | 306 kB 00:00 2021-11-11T20:46:29.616 INFO:teuthology.orchestra.run.smithi198.stdout:(9/23): python3-rados-17.0.0-8977.ga3a8018b.el8 3.4 MB/s | 366 kB 00:00 2021-11-11T20:46:29.674 INFO:teuthology.orchestra.run.smithi096.stdout:(13/23): leveldb-1.22-1.el8.x86_64.rpm 2.8 MB/s | 181 kB 00:00 2021-11-11T20:46:29.675 INFO:teuthology.orchestra.run.smithi198.stdout:(10/23): python3-rbd-17.0.0-8977.ga3a8018b.el8. 5.8 MB/s | 346 kB 00:00 2021-11-11T20:46:29.726 INFO:teuthology.orchestra.run.smithi198.stdout:(11/23): python3-rgw-17.0.0-8977.ga3a8018b.el8. 2.1 MB/s | 109 kB 00:00 2021-11-11T20:46:29.726 INFO:teuthology.orchestra.run.smithi096.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 1.1 MB/s | 59 kB 00:00 2021-11-11T20:46:29.752 INFO:teuthology.orchestra.run.smithi096.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 2.9 MB/s | 75 kB 00:00 2021-11-11T20:46:29.783 INFO:teuthology.orchestra.run.smithi096.stdout:(16/23): librdkafka-0.11.4-1.el8.x86_64.rpm 11 MB/s | 353 kB 00:00 2021-11-11T20:46:29.804 INFO:teuthology.orchestra.run.smithi096.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 12 MB/s | 259 kB 00:00 2021-11-11T20:46:29.872 INFO:teuthology.orchestra.run.smithi198.stdout:(12/23): gperftools-libs-2.7-9.el8.x86_64.rpm 2.0 MB/s | 306 kB 00:00 2021-11-11T20:46:29.899 INFO:teuthology.orchestra.run.smithi198.stdout:(13/23): leveldb-1.22-1.el8.x86_64.rpm 6.6 MB/s | 181 kB 00:00 2021-11-11T20:46:29.925 INFO:teuthology.orchestra.run.smithi198.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 2.3 MB/s | 59 kB 00:00 2021-11-11T20:46:29.935 INFO:teuthology.orchestra.run.smithi096.stdout:(18/23): ceph-common-17.0.0-8977.ga3a8018b.el8. 6.9 MB/s | 20 MB 00:02 2021-11-11T20:46:29.937 INFO:teuthology.orchestra.run.smithi096.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 331 kB/s | 44 kB 00:00 2021-11-11T20:46:29.952 INFO:teuthology.orchestra.run.smithi198.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 2.8 MB/s | 75 kB 00:00 2021-11-11T20:46:29.965 INFO:teuthology.orchestra.run.smithi096.stdout:(20/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 1.6 MB/s | 47 kB 00:00 2021-11-11T20:46:29.982 INFO:teuthology.orchestra.run.smithi198.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 1.5 MB/s | 47 kB 00:00 2021-11-11T20:46:30.004 INFO:teuthology.orchestra.run.smithi198.stdout:(17/23): librdkafka-0.11.4-1.el8.x86_64.rpm 16 MB/s | 353 kB 00:00 2021-11-11T20:46:30.027 INFO:teuthology.orchestra.run.smithi198.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 11 MB/s | 259 kB 00:00 2021-11-11T20:46:30.048 INFO:teuthology.orchestra.run.smithi198.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 2.0 MB/s | 44 kB 00:00 2021-11-11T20:46:30.363 INFO:teuthology.orchestra.run.smithi096.stdout:(21/23): librados2-17.0.0-8977.ga3a8018b.el8.x8 8.0 MB/s | 3.4 MB 00:00 2021-11-11T20:46:30.424 INFO:teuthology.orchestra.run.smithi198.stdout:(20/23): ceph-radosgw-17.0.0-8977.ga3a8018b.el8 2.8 MB/s | 9.1 MB 00:03 2021-11-11T20:46:30.512 INFO:teuthology.orchestra.run.smithi096.stdout:(22/23): librbd1-17.0.0-8977.ga3a8018b.el8.x86_ 7.1 MB/s | 3.9 MB 00:00 2021-11-11T20:46:30.537 INFO:teuthology.orchestra.run.smithi096.stdout:(23/23): ceph-radosgw-17.0.0-8977.ga3a8018b.el8 2.6 MB/s | 9.1 MB 00:03 2021-11-11T20:46:30.539 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:46:30.540 INFO:teuthology.orchestra.run.smithi096.stdout:Total 14 MB/s | 49 MB 00:03 2021-11-11T20:46:30.547 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:46:30.591 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:46:30.591 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:46:30.657 INFO:teuthology.orchestra.run.smithi198.stdout:(21/23): librados2-17.0.0-8977.ga3a8018b.el8.x8 5.6 MB/s | 3.4 MB 00:00 2021-11-11T20:46:30.797 INFO:teuthology.orchestra.run.smithi198.stdout:(22/23): ceph-common-17.0.0-8977.ga3a8018b.el8. 5.6 MB/s | 20 MB 00:03 2021-11-11T20:46:30.863 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:46:30.867 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:46:31.166 INFO:teuthology.orchestra.run.smithi198.stdout:(23/23): librbd1-17.0.0-8977.ga3a8018b.el8.x86_ 5.2 MB/s | 3.9 MB 00:00 2021-11-11T20:46:31.169 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:46:31.170 INFO:teuthology.orchestra.run.smithi198.stdout:Total 12 MB/s | 49 MB 00:03 2021-11-11T20:46:31.177 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:46:31.222 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:46:31.222 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:46:31.497 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:46:31.501 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:46:32.000 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:46:32.188 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2021-11-11T20:46:32.562 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2021-11-11T20:46:32.592 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:46:32.759 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2021-11-11T20:46:32.769 INFO:teuthology.orchestra.run.smithi096.stdout: Upgrading : librados2-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/25 2021-11-11T20:46:32.852 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librados2-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/25 2021-11-11T20:46:33.022 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-rados-2:17.0.0-8977.ga3a8018b.el8.x86_64 3/25 2021-11-11T20:46:33.179 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2021-11-11T20:46:33.269 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2021-11-11T20:46:33.384 INFO:teuthology.orchestra.run.smithi198.stdout: Upgrading : librados2-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/25 2021-11-11T20:46:33.479 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libcephfs2-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/25 2021-11-11T20:46:33.479 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: librados2-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/25 2021-11-11T20:46:33.665 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-rados-2:17.0.0-8977.ga3a8018b.el8.x86_64 3/25 2021-11-11T20:46:33.899 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2021-11-11T20:46:33.900 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: libcephfs2-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/25 2021-11-11T20:46:34.067 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : libcephfs2-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/25 2021-11-11T20:46:34.103 INFO:teuthology.orchestra.run.smithi096.stdout: Upgrading : librbd1-2:17.0.0-8977.ga3a8018b.el8.x86_64 6/25 2021-11-11T20:46:34.167 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librbd1-2:17.0.0-8977.ga3a8018b.el8.x86_64 6/25 2021-11-11T20:46:34.369 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2021-11-11T20:46:34.483 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: libcephfs2-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/25 2021-11-11T20:46:34.495 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : librdkafka-0.11.4-1.el8.x86_64 8/25 2021-11-11T20:46:34.668 INFO:teuthology.orchestra.run.smithi198.stdout: Upgrading : librbd1-2:17.0.0-8977.ga3a8018b.el8.x86_64 6/25 2021-11-11T20:46:34.769 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: librbd1-2:17.0.0-8977.ga3a8018b.el8.x86_64 6/25 2021-11-11T20:46:34.892 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : librdkafka-0.11.4-1.el8.x86_64 7/25 2021-11-11T20:46:34.923 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librdkafka-0.11.4-1.el8.x86_64 8/25 2021-11-11T20:46:34.942 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: librdkafka-0.11.4-1.el8.x86_64 7/25 2021-11-11T20:46:35.078 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : librgw2-2:17.0.0-8977.ga3a8018b.el8.x86_64 9/25 2021-11-11T20:46:35.153 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librgw2-2:17.0.0-8977.ga3a8018b.el8.x86_64 9/25 2021-11-11T20:46:35.326 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2021-11-11T20:46:35.454 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-ceph-argparse-2:17.0.0-8977.ga3a8018b.el8. 11/25 2021-11-11T20:46:35.467 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2021-11-11T20:46:35.640 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cephfs-2:17.0.0-8977.ga3a8018b.el8.x86_64 12/25 2021-11-11T20:46:35.666 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : librgw2-2:17.0.0-8977.ga3a8018b.el8.x86_64 9/25 2021-11-11T20:46:35.748 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: librgw2-2:17.0.0-8977.ga3a8018b.el8.x86_64 9/25 2021-11-11T20:46:35.870 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-rgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 13/25 2021-11-11T20:46:36.079 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-rbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 14/25 2021-11-11T20:46:36.118 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2021-11-11T20:46:36.227 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libradosstriper1-2:17.0.0-8977.ga3a8018b.el8.x86_6 15/25 2021-11-11T20:46:36.309 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-ceph-argparse-2:17.0.0-8977.ga3a8018b.el8. 11/25 2021-11-11T20:46:36.311 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: libradosstriper1-2:17.0.0-8977.ga3a8018b.el8.x86_6 15/25 2021-11-11T20:46:36.471 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2021-11-11T20:46:36.503 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-cephfs-2:17.0.0-8977.ga3a8018b.el8.x86_64 12/25 2021-11-11T20:46:36.653 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2021-11-11T20:46:36.693 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-rgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 13/25 2021-11-11T20:46:36.868 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2021-11-11T20:46:36.918 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-rbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 14/25 2021-11-11T20:46:37.036 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-ceph-common-2:17.0.0-8977.ga3a8018b.el8.x8 19/25 2021-11-11T20:46:37.083 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : libradosstriper1-2:17.0.0-8977.ga3a8018b.el8.x86_6 15/25 2021-11-11T20:46:37.151 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: libradosstriper1-2:17.0.0-8977.ga3a8018b.el8.x86_6 15/25 2021-11-11T20:46:37.326 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2021-11-11T20:46:37.524 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2021-11-11T20:46:37.720 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2021-11-11T20:46:37.892 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-ceph-common-2:17.0.0-8977.ga3a8018b.el8.x8 19/25 2021-11-11T20:46:41.153 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 20/25 2021-11-11T20:46:41.385 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 20/25 2021-11-11T20:46:41.959 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 20/25 2021-11-11T20:46:42.040 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 20/25 2021-11-11T20:46:42.199 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-base-2:17.0.0-8977.ga3a8018b.el8.x86_64 21/25 2021-11-11T20:46:42.274 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 20/25 2021-11-11T20:46:42.306 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-base-2:17.0.0-8977.ga3a8018b.el8.x86_64 21/25 2021-11-11T20:46:42.459 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-selinux-2:17.0.0-8977.ga3a8018b.el8.x86_64 22/25 2021-11-11T20:46:42.843 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 20/25 2021-11-11T20:46:43.095 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-base-2:17.0.0-8977.ga3a8018b.el8.x86_64 21/25 2021-11-11T20:46:43.203 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-base-2:17.0.0-8977.ga3a8018b.el8.x86_64 21/25 2021-11-11T20:46:43.349 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-selinux-2:17.0.0-8977.ga3a8018b.el8.x86_64 22/25 2021-11-11T20:47:00.169 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-selinux-2:17.0.0-8977.ga3a8018b.el8.x86_64 22/25 2021-11-11T20:47:00.170 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /sys 2021-11-11T20:47:00.170 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /proc 2021-11-11T20:47:00.170 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /dev 2021-11-11T20:47:00.170 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /run 2021-11-11T20:47:00.171 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /mnt 2021-11-11T20:47:00.171 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /var/tmp 2021-11-11T20:47:00.171 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /home 2021-11-11T20:47:00.171 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /tmp 2021-11-11T20:47:00.172 INFO:teuthology.orchestra.run.smithi096.stdout:skipping the directory /dev 2021-11-11T20:47:00.172 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:47:01.276 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-selinux-2:17.0.0-8977.ga3a8018b.el8.x86_64 22/25 2021-11-11T20:47:01.276 INFO:teuthology.orchestra.run.smithi198.stdout:skipping the directory /sys 2021-11-11T20:47:01.276 INFO:teuthology.orchestra.run.smithi198.stdout:skipping the directory /proc 2021-11-11T20:47:01.277 INFO:teuthology.orchestra.run.smithi198.stdout:skipping the directory /dev 2021-11-11T20:47:01.277 INFO:teuthology.orchestra.run.smithi198.stdout:skipping the directory /run 2021-11-11T20:47:01.278 INFO:teuthology.orchestra.run.smithi198.stdout:skipping the directory /mnt 2021-11-11T20:47:01.279 INFO:teuthology.orchestra.run.smithi198.stdout:skipping the directory /var/tmp 2021-11-11T20:47:01.279 INFO:teuthology.orchestra.run.smithi198.stdout:skipping the directory /home 2021-11-11T20:47:01.280 INFO:teuthology.orchestra.run.smithi198.stdout:skipping the directory /tmp 2021-11-11T20:47:01.280 INFO:teuthology.orchestra.run.smithi198.stdout:skipping the directory /dev 2021-11-11T20:47:01.281 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:47:01.335 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-radosgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 23/25 2021-11-11T20:47:01.420 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-radosgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 23/25 2021-11-11T20:47:01.420 INFO:teuthology.orchestra.run.smithi096.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2021-11-11T20:47:01.632 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2021-11-11T20:47:01.633 INFO:teuthology.orchestra.run.smithi096.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2021-11-11T20:47:02.361 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2021-11-11T20:47:02.362 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-base-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/25 2021-11-11T20:47:02.362 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/25 2021-11-11T20:47:02.363 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-radosgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 3/25 2021-11-11T20:47:02.363 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-selinux-2:17.0.0-8977.ga3a8018b.el8.x86_64 4/25 2021-11-11T20:47:02.364 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libcephfs2-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/25 2021-11-11T20:47:02.364 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libradosstriper1-2:17.0.0-8977.ga3a8018b.el8.x86_6 6/25 2021-11-11T20:47:02.364 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librgw2-2:17.0.0-8977.ga3a8018b.el8.x86_64 7/25 2021-11-11T20:47:02.364 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-ceph-argparse-2:17.0.0-8977.ga3a8018b.el8. 8/25 2021-11-11T20:47:02.365 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-ceph-common-2:17.0.0-8977.ga3a8018b.el8.x8 9/25 2021-11-11T20:47:02.365 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cephfs-2:17.0.0-8977.ga3a8018b.el8.x86_64 10/25 2021-11-11T20:47:02.367 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-rados-2:17.0.0-8977.ga3a8018b.el8.x86_64 11/25 2021-11-11T20:47:02.368 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-rbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 12/25 2021-11-11T20:47:02.368 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-rgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 13/25 2021-11-11T20:47:02.368 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2021-11-11T20:47:02.369 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2021-11-11T20:47:02.369 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2021-11-11T20:47:02.369 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2021-11-11T20:47:02.369 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librdkafka-0.11.4-1.el8.x86_64 18/25 2021-11-11T20:47:02.370 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2021-11-11T20:47:02.370 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2021-11-11T20:47:02.370 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2021-11-11T20:47:02.371 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librados2-2:17.0.0-8977.ga3a8018b.el8.x86_64 22/25 2021-11-11T20:47:02.371 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2021-11-11T20:47:02.371 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librbd1-2:17.0.0-8977.ga3a8018b.el8.x86_64 24/25 2021-11-11T20:47:02.417 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-radosgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 23/25 2021-11-11T20:47:02.497 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-radosgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 23/25 2021-11-11T20:47:02.498 INFO:teuthology.orchestra.run.smithi198.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2021-11-11T20:47:02.731 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2021-11-11T20:47:02.732 INFO:teuthology.orchestra.run.smithi198.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2021-11-11T20:47:03.110 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2021-11-11T20:47:03.110 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:47:03.477 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2021-11-11T20:47:03.477 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-base-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/25 2021-11-11T20:47:03.477 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/25 2021-11-11T20:47:03.478 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-radosgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 3/25 2021-11-11T20:47:03.478 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-selinux-2:17.0.0-8977.ga3a8018b.el8.x86_64 4/25 2021-11-11T20:47:03.478 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : libcephfs2-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/25 2021-11-11T20:47:03.478 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : libradosstriper1-2:17.0.0-8977.ga3a8018b.el8.x86_6 6/25 2021-11-11T20:47:03.478 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : librgw2-2:17.0.0-8977.ga3a8018b.el8.x86_64 7/25 2021-11-11T20:47:03.479 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-ceph-argparse-2:17.0.0-8977.ga3a8018b.el8. 8/25 2021-11-11T20:47:03.479 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-ceph-common-2:17.0.0-8977.ga3a8018b.el8.x8 9/25 2021-11-11T20:47:03.479 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-cephfs-2:17.0.0-8977.ga3a8018b.el8.x86_64 10/25 2021-11-11T20:47:03.479 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-rados-2:17.0.0-8977.ga3a8018b.el8.x86_64 11/25 2021-11-11T20:47:03.480 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-rbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 12/25 2021-11-11T20:47:03.480 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-rgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 13/25 2021-11-11T20:47:03.480 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2021-11-11T20:47:03.480 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2021-11-11T20:47:03.480 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2021-11-11T20:47:03.481 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2021-11-11T20:47:03.481 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2021-11-11T20:47:03.481 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : librdkafka-0.11.4-1.el8.x86_64 19/25 2021-11-11T20:47:03.481 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 20/25 2021-11-11T20:47:03.482 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 21/25 2021-11-11T20:47:03.483 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : librados2-2:17.0.0-8977.ga3a8018b.el8.x86_64 22/25 2021-11-11T20:47:03.483 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2021-11-11T20:47:03.483 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : librbd1-2:17.0.0-8977.ga3a8018b.el8.x86_64 24/25 2021-11-11T20:47:04.266 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2021-11-11T20:47:04.267 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:47:15.407 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:47:09 PM UTC. 2021-11-11T20:47:16.396 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:47:10 PM UTC. 2021-11-11T20:47:18.806 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:47:18.807 INFO:teuthology.orchestra.run.smithi096.stdout:Upgraded: 2021-11-11T20:47:18.807 INFO:teuthology.orchestra.run.smithi096.stdout: librados2-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.807 INFO:teuthology.orchestra.run.smithi096.stdout: librbd1-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.807 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:47:18.807 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-base-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.808 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.808 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-radosgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.808 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-selinux-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.808 INFO:teuthology.orchestra.run.smithi096.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2021-11-11T20:47:18.809 INFO:teuthology.orchestra.run.smithi096.stdout: leveldb-1.22-1.el8.x86_64 2021-11-11T20:47:18.809 INFO:teuthology.orchestra.run.smithi096.stdout: libcephfs2-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.809 INFO:teuthology.orchestra.run.smithi096.stdout: liboath-2.6.2-3.el8.x86_64 2021-11-11T20:47:18.809 INFO:teuthology.orchestra.run.smithi096.stdout: librabbitmq-0.9.0-3.el8.x86_64 2021-11-11T20:47:18.810 INFO:teuthology.orchestra.run.smithi096.stdout: libradosstriper1-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.810 INFO:teuthology.orchestra.run.smithi096.stdout: librdkafka-0.11.4-1.el8.x86_64 2021-11-11T20:47:18.810 INFO:teuthology.orchestra.run.smithi096.stdout: librgw2-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.810 INFO:teuthology.orchestra.run.smithi096.stdout: libunwind-1.3.1-3.el8.x86_64 2021-11-11T20:47:18.810 INFO:teuthology.orchestra.run.smithi096.stdout: lttng-ust-2.8.1-11.el8.x86_64 2021-11-11T20:47:18.811 INFO:teuthology.orchestra.run.smithi096.stdout: python3-ceph-argparse-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.811 INFO:teuthology.orchestra.run.smithi096.stdout: python3-ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.811 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cephfs-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.811 INFO:teuthology.orchestra.run.smithi096.stdout: python3-prettytable-0.7.2-14.el8.noarch 2021-11-11T20:47:18.812 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rados-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.812 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.812 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:18.812 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:47:18.812 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:47:18.935 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-test 2021-11-11T20:47:19.263 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:47:19.753 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 91 kB/s | 2.8 kB 00:00 2021-11-11T20:47:19.791 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:47:19.792 INFO:teuthology.orchestra.run.smithi198.stdout:Upgraded: 2021-11-11T20:47:19.792 INFO:teuthology.orchestra.run.smithi198.stdout: librados2-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.792 INFO:teuthology.orchestra.run.smithi198.stdout: librbd1-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.793 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:47:19.793 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-base-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.793 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.793 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-radosgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.794 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-selinux-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.794 INFO:teuthology.orchestra.run.smithi198.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2021-11-11T20:47:19.794 INFO:teuthology.orchestra.run.smithi198.stdout: leveldb-1.22-1.el8.x86_64 2021-11-11T20:47:19.795 INFO:teuthology.orchestra.run.smithi198.stdout: libcephfs2-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.795 INFO:teuthology.orchestra.run.smithi198.stdout: liboath-2.6.2-3.el8.x86_64 2021-11-11T20:47:19.795 INFO:teuthology.orchestra.run.smithi198.stdout: librabbitmq-0.9.0-3.el8.x86_64 2021-11-11T20:47:19.795 INFO:teuthology.orchestra.run.smithi198.stdout: libradosstriper1-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.796 INFO:teuthology.orchestra.run.smithi198.stdout: librdkafka-0.11.4-1.el8.x86_64 2021-11-11T20:47:19.796 INFO:teuthology.orchestra.run.smithi198.stdout: librgw2-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.796 INFO:teuthology.orchestra.run.smithi198.stdout: libunwind-1.3.1-3.el8.x86_64 2021-11-11T20:47:19.796 INFO:teuthology.orchestra.run.smithi198.stdout: lttng-ust-2.8.1-11.el8.x86_64 2021-11-11T20:47:19.796 INFO:teuthology.orchestra.run.smithi198.stdout: python3-ceph-argparse-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.797 INFO:teuthology.orchestra.run.smithi198.stdout: python3-ceph-common-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.797 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cephfs-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.797 INFO:teuthology.orchestra.run.smithi198.stdout: python3-prettytable-0.7.2-14.el8.noarch 2021-11-11T20:47:19.798 INFO:teuthology.orchestra.run.smithi198.stdout: python3-rados-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.798 INFO:teuthology.orchestra.run.smithi198.stdout: python3-rbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.798 INFO:teuthology.orchestra.run.smithi198.stdout: python3-rgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:47:19.798 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:47:19.799 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:47:19.934 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph-test 2021-11-11T20:47:20.263 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:47:20.742 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 82 kB/s | 2.4 kB 00:00 2021-11-11T20:47:24.670 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 80 kB/s | 2.4 kB 00:00 2021-11-11T20:47:26.027 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 86 kB/s | 2.8 kB 00:00 2021-11-11T20:47:30.119 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:47:31.086 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:47:32.675 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:47:32.677 INFO:teuthology.orchestra.run.smithi096.stdout:============================================================================================ 2021-11-11T20:47:32.678 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2021-11-11T20:47:32.678 INFO:teuthology.orchestra.run.smithi096.stdout:============================================================================================ 2021-11-11T20:47:32.679 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:47:32.679 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-test x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 45 M 2021-11-11T20:47:32.679 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2021-11-11T20:47:32.679 INFO:teuthology.orchestra.run.smithi096.stdout: jq x86_64 1.5-12.el8 rhel-8-for-x86_64-appstream-rpms 161 k 2021-11-11T20:47:32.680 INFO:teuthology.orchestra.run.smithi096.stdout: libcephsqlite x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 170 k 2021-11-11T20:47:32.680 INFO:teuthology.orchestra.run.smithi096.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2021-11-11T20:47:32.680 INFO:teuthology.orchestra.run.smithi096.stdout: socat x86_64 1.7.3.3-2.el8 rhel-8-for-x86_64-appstream-rpms 302 k 2021-11-11T20:47:32.680 INFO:teuthology.orchestra.run.smithi096.stdout: xmlstarlet x86_64 1.6.1-11.el8 epel 69 k 2021-11-11T20:47:32.680 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:47:32.681 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:47:32.681 INFO:teuthology.orchestra.run.smithi096.stdout:============================================================================================ 2021-11-11T20:47:32.681 INFO:teuthology.orchestra.run.smithi096.stdout:Install 6 Packages 2021-11-11T20:47:32.681 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:47:32.682 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 45 M 2021-11-11T20:47:32.682 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 211 M 2021-11-11T20:47:32.682 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:47:32.927 INFO:teuthology.orchestra.run.smithi096.stdout:(1/6): xmlstarlet-1.6.1-11.el8.x86_64.rpm 278 kB/s | 69 kB 00:00 2021-11-11T20:47:32.961 INFO:teuthology.orchestra.run.smithi096.stdout:(2/6): jq-1.5-12.el8.x86_64.rpm 4.6 MB/s | 161 kB 00:00 2021-11-11T20:47:32.983 INFO:teuthology.orchestra.run.smithi096.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 8.5 MB/s | 187 kB 00:00 2021-11-11T20:47:33.005 INFO:teuthology.orchestra.run.smithi096.stdout:(4/6): socat-1.7.3.3-2.el8.x86_64.rpm 13 MB/s | 302 kB 00:00 2021-11-11T20:47:33.165 INFO:teuthology.orchestra.run.smithi096.stdout:(5/6): libcephsqlite-17.0.0-8977.ga3a8018b.el8. 349 kB/s | 170 kB 00:00 2021-11-11T20:47:33.680 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:47:33.684 INFO:teuthology.orchestra.run.smithi198.stdout:============================================================================================ 2021-11-11T20:47:33.685 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repository Size 2021-11-11T20:47:33.685 INFO:teuthology.orchestra.run.smithi198.stdout:============================================================================================ 2021-11-11T20:47:33.687 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:47:33.687 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-test x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 45 M 2021-11-11T20:47:33.688 INFO:teuthology.orchestra.run.smithi198.stdout:Installing dependencies: 2021-11-11T20:47:33.688 INFO:teuthology.orchestra.run.smithi198.stdout: jq x86_64 1.5-12.el8 rhel-8-for-x86_64-appstream-rpms 161 k 2021-11-11T20:47:33.689 INFO:teuthology.orchestra.run.smithi198.stdout: libcephsqlite x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 170 k 2021-11-11T20:47:33.689 INFO:teuthology.orchestra.run.smithi198.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2021-11-11T20:47:33.690 INFO:teuthology.orchestra.run.smithi198.stdout: socat x86_64 1.7.3.3-2.el8 rhel-8-for-x86_64-appstream-rpms 302 k 2021-11-11T20:47:33.690 INFO:teuthology.orchestra.run.smithi198.stdout: xmlstarlet x86_64 1.6.1-11.el8 epel 69 k 2021-11-11T20:47:33.690 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:47:33.691 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:47:33.692 INFO:teuthology.orchestra.run.smithi198.stdout:============================================================================================ 2021-11-11T20:47:33.693 INFO:teuthology.orchestra.run.smithi198.stdout:Install 6 Packages 2021-11-11T20:47:33.694 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:47:33.695 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 45 M 2021-11-11T20:47:33.695 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 211 M 2021-11-11T20:47:33.696 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:47:34.323 INFO:teuthology.orchestra.run.smithi198.stdout:(1/6): xmlstarlet-1.6.1-11.el8.x86_64.rpm 108 kB/s | 69 kB 00:00 2021-11-11T20:47:34.353 INFO:teuthology.orchestra.run.smithi198.stdout:(2/6): jq-1.5-12.el8.x86_64.rpm 4.9 MB/s | 161 kB 00:00 2021-11-11T20:47:34.374 INFO:teuthology.orchestra.run.smithi198.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 8.9 MB/s | 187 kB 00:00 2021-11-11T20:47:34.395 INFO:teuthology.orchestra.run.smithi198.stdout:(4/6): socat-1.7.3.3-2.el8.x86_64.rpm 14 MB/s | 302 kB 00:00 2021-11-11T20:47:34.521 INFO:teuthology.orchestra.run.smithi198.stdout:(5/6): libcephsqlite-17.0.0-8977.ga3a8018b.el8. 203 kB/s | 170 kB 00:00 2021-11-11T20:47:41.378 INFO:teuthology.orchestra.run.smithi198.stdout:(6/6): ceph-test-17.0.0-8977.ga3a8018b.el8.x86_ 5.8 MB/s | 45 MB 00:07 2021-11-11T20:47:41.380 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:47:41.381 INFO:teuthology.orchestra.run.smithi198.stdout:Total 5.9 MB/s | 45 MB 00:07 2021-11-11T20:47:41.386 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:47:41.409 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:47:41.409 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:47:41.676 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:47:41.680 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:47:42.158 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:47:42.492 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : socat-1.7.3.3-2.el8.x86_64 1/6 2021-11-11T20:47:42.681 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 2/6 2021-11-11T20:47:42.759 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 2/6 2021-11-11T20:47:42.940 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : jq-1.5-12.el8.x86_64 3/6 2021-11-11T20:47:43.095 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : xmlstarlet-1.6.1-11.el8.x86_64 4/6 2021-11-11T20:47:43.247 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : libcephsqlite-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/6 2021-11-11T20:47:47.594 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: libcephsqlite-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/6 2021-11-11T20:47:47.835 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-test-2:17.0.0-8977.ga3a8018b.el8.x86_64 6/6 2021-11-11T20:47:48.051 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-test-2:17.0.0-8977.ga3a8018b.el8.x86_64 6/6 2021-11-11T20:47:48.051 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-test-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/6 2021-11-11T20:47:48.051 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : libcephsqlite-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/6 2021-11-11T20:47:48.052 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : xmlstarlet-1.6.1-11.el8.x86_64 3/6 2021-11-11T20:47:48.052 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : jq-1.5-12.el8.x86_64 4/6 2021-11-11T20:47:48.052 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2021-11-11T20:47:48.397 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : socat-1.7.3.3-2.el8.x86_64 6/6 2021-11-11T20:47:48.397 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:47:50.246 INFO:teuthology.orchestra.run.smithi096.stdout:(6/6): ceph-test-17.0.0-8977.ga3a8018b.el8.x86_ 2.5 MB/s | 45 MB 00:17 2021-11-11T20:47:50.249 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:47:50.249 INFO:teuthology.orchestra.run.smithi096.stdout:Total 2.6 MB/s | 45 MB 00:17 2021-11-11T20:47:50.256 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:47:50.279 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:47:50.280 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:47:50.530 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:47:50.534 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:47:51.026 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:47:51.189 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : socat-1.7.3.3-2.el8.x86_64 1/6 2021-11-11T20:47:51.386 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 2/6 2021-11-11T20:47:51.450 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 2/6 2021-11-11T20:47:51.604 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : jq-1.5-12.el8.x86_64 3/6 2021-11-11T20:47:51.775 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : xmlstarlet-1.6.1-11.el8.x86_64 4/6 2021-11-11T20:47:51.901 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libcephsqlite-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/6 2021-11-11T20:47:56.259 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: libcephsqlite-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/6 2021-11-11T20:47:56.515 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-test-2:17.0.0-8977.ga3a8018b.el8.x86_64 6/6 2021-11-11T20:47:56.731 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-test-2:17.0.0-8977.ga3a8018b.el8.x86_64 6/6 2021-11-11T20:47:56.731 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-test-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/6 2021-11-11T20:47:56.732 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libcephsqlite-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/6 2021-11-11T20:47:56.732 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : xmlstarlet-1.6.1-11.el8.x86_64 3/6 2021-11-11T20:47:56.732 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : jq-1.5-12.el8.x86_64 4/6 2021-11-11T20:47:56.732 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2021-11-11T20:47:57.051 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : socat-1.7.3.3-2.el8.x86_64 6/6 2021-11-11T20:47:57.052 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:48:00.239 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:47:10 PM UTC. 2021-11-11T20:48:02.987 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:48:02.988 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:48:02.988 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-test-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:02.988 INFO:teuthology.orchestra.run.smithi198.stdout: jq-1.5-12.el8.x86_64 2021-11-11T20:48:02.988 INFO:teuthology.orchestra.run.smithi198.stdout: libcephsqlite-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:02.989 INFO:teuthology.orchestra.run.smithi198.stdout: oniguruma-6.8.2-2.el8.x86_64 2021-11-11T20:48:02.989 INFO:teuthology.orchestra.run.smithi198.stdout: socat-1.7.3.3-2.el8.x86_64 2021-11-11T20:48:02.989 INFO:teuthology.orchestra.run.smithi198.stdout: xmlstarlet-1.6.1-11.el8.x86_64 2021-11-11T20:48:02.989 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:48:02.990 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:48:03.133 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph 2021-11-11T20:48:03.452 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:48:04.028 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 81 kB/s | 2.4 kB 00:00 2021-11-11T20:48:09.338 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:47:09 PM UTC. 2021-11-11T20:48:09.368 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 91 kB/s | 2.8 kB 00:00 2021-11-11T20:48:12.058 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:48:12.059 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:48:12.059 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-test-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:12.059 INFO:teuthology.orchestra.run.smithi096.stdout: jq-1.5-12.el8.x86_64 2021-11-11T20:48:12.059 INFO:teuthology.orchestra.run.smithi096.stdout: libcephsqlite-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:12.060 INFO:teuthology.orchestra.run.smithi096.stdout: oniguruma-6.8.2-2.el8.x86_64 2021-11-11T20:48:12.060 INFO:teuthology.orchestra.run.smithi096.stdout: socat-1.7.3.3-2.el8.x86_64 2021-11-11T20:48:12.060 INFO:teuthology.orchestra.run.smithi096.stdout: xmlstarlet-1.6.1-11.el8.x86_64 2021-11-11T20:48:12.060 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:48:12.061 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:48:12.201 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph 2021-11-11T20:48:12.517 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:48:13.034 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 95 kB/s | 2.8 kB 00:00 2021-11-11T20:48:14.439 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:48:17.052 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:48:17.056 INFO:teuthology.orchestra.run.smithi198.stdout:======================================================================================================= 2021-11-11T20:48:17.056 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repository Size 2021-11-11T20:48:17.057 INFO:teuthology.orchestra.run.smithi198.stdout:======================================================================================================= 2021-11-11T20:48:17.057 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:48:17.057 INFO:teuthology.orchestra.run.smithi198.stdout: ceph x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 6.4 k 2021-11-11T20:48:17.058 INFO:teuthology.orchestra.run.smithi198.stdout:Installing dependencies: 2021-11-11T20:48:17.058 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mds x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 2.3 M 2021-11-11T20:48:17.058 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 1.5 M 2021-11-11T20:48:17.059 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr-modules-core noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 216 k 2021-11-11T20:48:17.059 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mon x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 3.9 M 2021-11-11T20:48:17.059 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-osd x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 17 M 2021-11-11T20:48:17.059 INFO:teuthology.orchestra.run.smithi198.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2021-11-11T20:48:17.059 INFO:teuthology.orchestra.run.smithi198.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2021-11-11T20:48:17.060 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2021-11-11T20:48:17.060 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2021-11-11T20:48:17.060 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2021-11-11T20:48:17.060 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cryptography x86_64 3.2.1-4.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2021-11-11T20:48:17.061 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2021-11-11T20:48:17.061 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2021-11-11T20:48:17.061 INFO:teuthology.orchestra.run.smithi198.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2021-11-11T20:48:17.061 INFO:teuthology.orchestra.run.smithi198.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2021-11-11T20:48:17.061 INFO:teuthology.orchestra.run.smithi198.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2021-11-11T20:48:17.062 INFO:teuthology.orchestra.run.smithi198.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2021-11-11T20:48:17.062 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2021-11-11T20:48:17.062 INFO:teuthology.orchestra.run.smithi198.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2021-11-11T20:48:17.062 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2021-11-11T20:48:17.063 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2021-11-11T20:48:17.063 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2021-11-11T20:48:17.063 INFO:teuthology.orchestra.run.smithi198.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2021-11-11T20:48:17.063 INFO:teuthology.orchestra.run.smithi198.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2021-11-11T20:48:17.064 INFO:teuthology.orchestra.run.smithi198.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2021-11-11T20:48:17.064 INFO:teuthology.orchestra.run.smithi198.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2021-11-11T20:48:17.064 INFO:teuthology.orchestra.run.smithi198.stdout: python3-waitress noarch 1.2.1-2.el8.1 epel 241 k 2021-11-11T20:48:17.064 INFO:teuthology.orchestra.run.smithi198.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2021-11-11T20:48:17.065 INFO:teuthology.orchestra.run.smithi198.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2021-11-11T20:48:17.065 INFO:teuthology.orchestra.run.smithi198.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2021-11-11T20:48:17.066 INFO:teuthology.orchestra.run.smithi198.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2021-11-11T20:48:17.066 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:48:17.066 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:48:17.067 INFO:teuthology.orchestra.run.smithi198.stdout:======================================================================================================= 2021-11-11T20:48:17.067 INFO:teuthology.orchestra.run.smithi198.stdout:Install 32 Packages 2021-11-11T20:48:17.067 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:48:17.068 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 28 M 2021-11-11T20:48:17.069 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 112 M 2021-11-11T20:48:17.069 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:48:17.202 INFO:teuthology.orchestra.run.smithi198.stdout:(1/32): ceph-17.0.0-8977.ga3a8018b.el8.x86_64.r 45 kB/s | 6.4 kB 00:00 2021-11-11T20:48:17.547 INFO:teuthology.orchestra.run.smithi198.stdout:(2/32): ceph-mgr-17.0.0-8977.ga3a8018b.el8.x86_ 3.1 MB/s | 1.5 MB 00:00 2021-11-11T20:48:17.565 INFO:teuthology.orchestra.run.smithi198.stdout:(3/32): ceph-mds-17.0.0-8977.ga3a8018b.el8.x86_ 4.5 MB/s | 2.3 MB 00:00 2021-11-11T20:48:17.625 INFO:teuthology.orchestra.run.smithi198.stdout:(4/32): ceph-mgr-modules-core-17.0.0-8977.ga3a8 3.5 MB/s | 216 kB 00:00 2021-11-11T20:48:17.678 INFO:teuthology.orchestra.run.smithi198.stdout:(5/32): ceph-mon-17.0.0-8977.ga3a8018b.el8.x86_ 8.1 MB/s | 3.9 MB 00:00 2021-11-11T20:48:17.719 INFO:teuthology.orchestra.run.smithi198.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 464 kB/s | 44 kB 00:00 2021-11-11T20:48:17.789 INFO:teuthology.orchestra.run.smithi198.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 2.4 MB/s | 173 kB 00:00 2021-11-11T20:48:17.823 INFO:teuthology.orchestra.run.smithi198.stdout:(8/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.3 MB/s | 185 kB 00:00 2021-11-11T20:48:17.840 INFO:teuthology.orchestra.run.smithi198.stdout:(9/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 7.4 MB/s | 384 kB 00:00 2021-11-11T20:48:17.850 INFO:teuthology.orchestra.run.smithi198.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 397 kB/s | 11 kB 00:00 2021-11-11T20:48:17.868 INFO:teuthology.orchestra.run.smithi198.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 733 kB/s | 18 kB 00:00 2021-11-11T20:48:17.880 INFO:teuthology.orchestra.run.smithi198.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 1.6 MB/s | 49 kB 00:00 2021-11-11T20:48:17.890 INFO:teuthology.orchestra.run.smithi198.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 2.4 MB/s | 59 kB 00:00 2021-11-11T20:48:17.912 INFO:teuthology.orchestra.run.smithi198.stdout:(14/32): python3-portend-2.6-1.el8.noarch.rpm 664 kB/s | 16 kB 00:00 2021-11-11T20:48:17.936 INFO:teuthology.orchestra.run.smithi198.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 4.9 MB/s | 283 kB 00:00 2021-11-11T20:48:17.937 INFO:teuthology.orchestra.run.smithi198.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 754 kB/s | 19 kB 00:00 2021-11-11T20:48:17.961 INFO:teuthology.orchestra.run.smithi198.stdout:(17/32): python3-tempora-1.14.1-5.el8.noarch.rp 1.2 MB/s | 29 kB 00:00 2021-11-11T20:48:17.968 INFO:teuthology.orchestra.run.smithi198.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 807 kB/s | 24 kB 00:00 2021-11-11T20:48:17.986 INFO:teuthology.orchestra.run.smithi198.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 1.1 MB/s | 27 kB 00:00 2021-11-11T20:48:17.998 INFO:teuthology.orchestra.run.smithi198.stdout:(20/32): python3-waitress-1.2.1-2.el8.1.noarch. 7.5 MB/s | 241 kB 00:00 2021-11-11T20:48:18.012 INFO:teuthology.orchestra.run.smithi198.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 9.2 MB/s | 251 kB 00:00 2021-11-11T20:48:18.029 INFO:teuthology.orchestra.run.smithi198.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 2.7 MB/s | 85 kB 00:00 2021-11-11T20:48:18.040 INFO:teuthology.orchestra.run.smithi198.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 895 kB/s | 23 kB 00:00 2021-11-11T20:48:18.068 INFO:teuthology.orchestra.run.smithi198.stdout:(24/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 6.2 MB/s | 238 kB 00:00 2021-11-11T20:48:18.073 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 69 kB/s | 2.4 kB 00:00 2021-11-11T20:48:18.082 INFO:teuthology.orchestra.run.smithi198.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 2.6 MB/s | 109 kB 00:00 2021-11-11T20:48:18.100 INFO:teuthology.orchestra.run.smithi198.stdout:(26/32): python3-cryptography-3.2.1-4.el8.x86_6 18 MB/s | 559 kB 00:00 2021-11-11T20:48:18.108 INFO:teuthology.orchestra.run.smithi198.stdout:(27/32): python3-mako-1.0.6-13.el8.noarch.rpm 5.4 MB/s | 157 kB 00:00 2021-11-11T20:48:18.125 INFO:teuthology.orchestra.run.smithi198.stdout:(28/32): python3-pytz-2017.2-9.el8.noarch.rpm 2.2 MB/s | 54 kB 00:00 2021-11-11T20:48:18.132 INFO:teuthology.orchestra.run.smithi198.stdout:(29/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 4.2 MB/s | 103 kB 00:00 2021-11-11T20:48:18.152 INFO:teuthology.orchestra.run.smithi198.stdout:(30/32): python3-werkzeug-0.12.2-4.el8.noarch.r 17 MB/s | 457 kB 00:00 2021-11-11T20:48:18.156 INFO:teuthology.orchestra.run.smithi198.stdout:(31/32): python3-markupsafe-0.23-19.el8.x86_64. 1.6 MB/s | 39 kB 00:00 2021-11-11T20:48:18.330 INFO:teuthology.orchestra.run.smithi198.stdout:(32/32): ceph-osd-17.0.0-8977.ga3a8018b.el8.x86 21 MB/s | 17 MB 00:00 2021-11-11T20:48:18.334 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:48:18.334 INFO:teuthology.orchestra.run.smithi198.stdout:Total 22 MB/s | 28 MB 00:01 2021-11-11T20:48:18.348 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:48:18.385 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:48:18.386 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:48:18.601 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:48:18.602 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:48:21.545 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:48:21.783 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2021-11-11T20:48:21.970 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2021-11-11T20:48:22.248 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 3/32 2021-11-11T20:48:22.551 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 4/32 2021-11-11T20:48:22.781 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 5/32 2021-11-11T20:48:22.981 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2021-11-11T20:48:23.264 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2021-11-11T20:48:23.485 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:04 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:48:23.659 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2021-11-11T20:48:23.919 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-cryptography-3.2.1-4.el8.x86_64 9/32 2021-11-11T20:48:24.098 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2021-11-11T20:48:24.300 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2021-11-11T20:48:24.516 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2021-11-11T20:48:24.742 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2021-11-11T20:48:24.947 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-waitress-1.2.1-2.el8.1.noarch 14/32 2021-11-11T20:48:25.170 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2021-11-11T20:48:25.388 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2021-11-11T20:48:25.569 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2021-11-11T20:48:25.777 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2021-11-11T20:48:26.045 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2021-11-11T20:48:26.095 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:48:26.098 INFO:teuthology.orchestra.run.smithi096.stdout:======================================================================================================= 2021-11-11T20:48:26.098 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2021-11-11T20:48:26.098 INFO:teuthology.orchestra.run.smithi096.stdout:======================================================================================================= 2021-11-11T20:48:26.099 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:48:26.099 INFO:teuthology.orchestra.run.smithi096.stdout: ceph x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 6.4 k 2021-11-11T20:48:26.099 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2021-11-11T20:48:26.100 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mds x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 2.3 M 2021-11-11T20:48:26.100 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 1.5 M 2021-11-11T20:48:26.101 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-modules-core noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 216 k 2021-11-11T20:48:26.101 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mon x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 3.9 M 2021-11-11T20:48:26.101 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-osd x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 17 M 2021-11-11T20:48:26.101 INFO:teuthology.orchestra.run.smithi096.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2021-11-11T20:48:26.102 INFO:teuthology.orchestra.run.smithi096.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2021-11-11T20:48:26.102 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2021-11-11T20:48:26.102 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2021-11-11T20:48:26.103 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2021-11-11T20:48:26.103 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cryptography x86_64 3.2.1-4.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2021-11-11T20:48:26.103 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2021-11-11T20:48:26.103 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2021-11-11T20:48:26.104 INFO:teuthology.orchestra.run.smithi096.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2021-11-11T20:48:26.104 INFO:teuthology.orchestra.run.smithi096.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2021-11-11T20:48:26.104 INFO:teuthology.orchestra.run.smithi096.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2021-11-11T20:48:26.104 INFO:teuthology.orchestra.run.smithi096.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2021-11-11T20:48:26.105 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2021-11-11T20:48:26.105 INFO:teuthology.orchestra.run.smithi096.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2021-11-11T20:48:26.105 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2021-11-11T20:48:26.105 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2021-11-11T20:48:26.106 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2021-11-11T20:48:26.106 INFO:teuthology.orchestra.run.smithi096.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2021-11-11T20:48:26.106 INFO:teuthology.orchestra.run.smithi096.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2021-11-11T20:48:26.106 INFO:teuthology.orchestra.run.smithi096.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2021-11-11T20:48:26.107 INFO:teuthology.orchestra.run.smithi096.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2021-11-11T20:48:26.107 INFO:teuthology.orchestra.run.smithi096.stdout: python3-waitress noarch 1.2.1-2.el8.1 epel 241 k 2021-11-11T20:48:26.107 INFO:teuthology.orchestra.run.smithi096.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2021-11-11T20:48:26.107 INFO:teuthology.orchestra.run.smithi096.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2021-11-11T20:48:26.108 INFO:teuthology.orchestra.run.smithi096.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2021-11-11T20:48:26.108 INFO:teuthology.orchestra.run.smithi096.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2021-11-11T20:48:26.108 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:48:26.108 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:48:26.109 INFO:teuthology.orchestra.run.smithi096.stdout:======================================================================================================= 2021-11-11T20:48:26.109 INFO:teuthology.orchestra.run.smithi096.stdout:Install 32 Packages 2021-11-11T20:48:26.109 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:48:26.110 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 28 M 2021-11-11T20:48:26.111 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 112 M 2021-11-11T20:48:26.111 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:48:26.245 INFO:teuthology.orchestra.run.smithi096.stdout:(1/32): ceph-17.0.0-8977.ga3a8018b.el8.x86_64.r 45 kB/s | 6.4 kB 00:00 2021-11-11T20:48:26.307 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2021-11-11T20:48:26.504 INFO:teuthology.orchestra.run.smithi096.stdout:(2/32): ceph-mgr-17.0.0-8977.ga3a8018b.el8.x86_ 3.8 MB/s | 1.5 MB 00:00 2021-11-11T20:48:26.537 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2021-11-11T20:48:26.559 INFO:teuthology.orchestra.run.smithi096.stdout:(3/32): ceph-mds-17.0.0-8977.ga3a8018b.el8.x86_ 5.0 MB/s | 2.3 MB 00:00 2021-11-11T20:48:26.626 INFO:teuthology.orchestra.run.smithi096.stdout:(4/32): ceph-mgr-modules-core-17.0.0-8977.ga3a8 3.2 MB/s | 216 kB 00:00 2021-11-11T20:48:26.740 INFO:teuthology.orchestra.run.smithi096.stdout:(5/32): ceph-mon-17.0.0-8977.ga3a8018b.el8.x86_ 7.8 MB/s | 3.9 MB 00:00 2021-11-11T20:48:26.740 INFO:teuthology.orchestra.run.smithi096.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 382 kB/s | 44 kB 00:00 2021-11-11T20:48:26.773 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2021-11-11T20:48:26.820 INFO:teuthology.orchestra.run.smithi096.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 2.2 MB/s | 185 kB 00:00 2021-11-11T20:48:26.861 INFO:teuthology.orchestra.run.smithi096.stdout:(8/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 1.4 MB/s | 173 kB 00:00 2021-11-11T20:48:26.876 INFO:teuthology.orchestra.run.smithi096.stdout:(9/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 6.7 MB/s | 384 kB 00:00 2021-11-11T20:48:26.886 INFO:teuthology.orchestra.run.smithi096.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 455 kB/s | 11 kB 00:00 2021-11-11T20:48:26.903 INFO:teuthology.orchestra.run.smithi096.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 651 kB/s | 18 kB 00:00 2021-11-11T20:48:26.911 INFO:teuthology.orchestra.run.smithi096.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 1.9 MB/s | 49 kB 00:00 2021-11-11T20:48:26.931 INFO:teuthology.orchestra.run.smithi096.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 2.1 MB/s | 59 kB 00:00 2021-11-11T20:48:26.958 INFO:teuthology.orchestra.run.smithi096.stdout:(14/32): python3-portend-2.6-1.el8.noarch.rpm 579 kB/s | 16 kB 00:00 2021-11-11T20:48:26.961 INFO:teuthology.orchestra.run.smithi096.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 5.5 MB/s | 283 kB 00:00 2021-11-11T20:48:26.985 INFO:teuthology.orchestra.run.smithi096.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 695 kB/s | 19 kB 00:00 2021-11-11T20:48:26.986 INFO:teuthology.orchestra.run.smithi096.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 979 kB/s | 24 kB 00:00 2021-11-11T20:48:27.011 INFO:teuthology.orchestra.run.smithi096.stdout:(18/32): python3-trustme-0.6.0-4.el8.noarch.rpm 1.1 MB/s | 27 kB 00:00 2021-11-11T20:48:27.013 INFO:teuthology.orchestra.run.smithi096.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 1.0 MB/s | 29 kB 00:00 2021-11-11T20:48:27.013 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2021-11-11T20:48:27.038 INFO:teuthology.orchestra.run.smithi096.stdout:(20/32): python3-waitress-1.2.1-2.el8.1.noarch. 8.7 MB/s | 241 kB 00:00 2021-11-11T20:48:27.043 INFO:teuthology.orchestra.run.smithi096.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 8.1 MB/s | 251 kB 00:00 2021-11-11T20:48:27.064 INFO:teuthology.orchestra.run.smithi096.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 3.3 MB/s | 85 kB 00:00 2021-11-11T20:48:27.072 INFO:teuthology.orchestra.run.smithi096.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 790 kB/s | 23 kB 00:00 2021-11-11T20:48:27.100 INFO:teuthology.orchestra.run.smithi096.stdout:(24/32): python3-mako-1.0.6-13.el8.noarch.rpm 4.3 MB/s | 157 kB 00:00 2021-11-11T20:48:27.112 INFO:teuthology.orchestra.run.smithi096.stdout:(25/32): python3-pytz-2017.2-9.el8.noarch.rpm 1.4 MB/s | 54 kB 00:00 2021-11-11T20:48:27.130 INFO:teuthology.orchestra.run.smithi096.stdout:(26/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 3.9 MB/s | 103 kB 00:00 2021-11-11T20:48:27.140 INFO:teuthology.orchestra.run.smithi096.stdout:(27/32): python3-werkzeug-0.12.2-4.el8.noarch.r 16 MB/s | 457 kB 00:00 2021-11-11T20:48:27.151 INFO:teuthology.orchestra.run.smithi096.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 1.8 MB/s | 39 kB 00:00 2021-11-11T20:48:27.170 INFO:teuthology.orchestra.run.smithi096.stdout:(29/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 9.0 MB/s | 238 kB 00:00 2021-11-11T20:48:27.178 INFO:teuthology.orchestra.run.smithi096.stdout:(30/32): python3-pycparser-2.14-14.el8.noarch.r 4.1 MB/s | 109 kB 00:00 2021-11-11T20:48:27.194 INFO:teuthology.orchestra.run.smithi096.stdout:(31/32): python3-cryptography-3.2.1-4.el8.x86_6 23 MB/s | 559 kB 00:00 2021-11-11T20:48:27.226 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2021-11-11T20:48:27.395 INFO:teuthology.orchestra.run.smithi096.stdout:(32/32): ceph-osd-17.0.0-8977.ga3a8018b.el8.x86 19 MB/s | 17 MB 00:00 2021-11-11T20:48:27.396 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:48:27.397 INFO:teuthology.orchestra.run.smithi096.stdout:Total 22 MB/s | 28 MB 00:01 2021-11-11T20:48:27.411 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:48:27.446 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:48:27.447 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:48:27.566 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2021-11-11T20:48:27.647 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:48:27.651 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:48:27.881 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2021-11-11T20:48:28.279 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-mgr-modules-core-2:17.0.0-8977.ga3a8018b.el8. 27/32 2021-11-11T20:48:28.495 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-mgr-2:17.0.0-8977.ga3a8018b.el8.x86_64 28/32 2021-11-11T20:48:29.973 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-mgr-2:17.0.0-8977.ga3a8018b.el8.x86_64 28/32 2021-11-11T20:48:30.185 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-osd-2:17.0.0-8977.ga3a8018b.el8.x86_64 29/32 2021-11-11T20:48:30.409 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:48:30.577 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-osd-2:17.0.0-8977.ga3a8018b.el8.x86_64 29/32 2021-11-11T20:48:30.653 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2021-11-11T20:48:30.756 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-mon-2:17.0.0-8977.ga3a8018b.el8.x86_64 30/32 2021-11-11T20:48:30.891 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2021-11-11T20:48:30.974 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-mon-2:17.0.0-8977.ga3a8018b.el8.x86_64 30/32 2021-11-11T20:48:31.140 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2021-11-11T20:48:31.148 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-mds-2:17.0.0-8977.ga3a8018b.el8.x86_64 31/32 2021-11-11T20:48:31.172 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-mds-2:17.0.0-8977.ga3a8018b.el8.x86_64 31/32 2021-11-11T20:48:31.262 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-2:17.0.0-8977.ga3a8018b.el8.x86_64 32/32 2021-11-11T20:48:31.536 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2021-11-11T20:48:31.598 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-2:17.0.0-8977.ga3a8018b.el8.x86_64 32/32 2021-11-11T20:48:31.598 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/32 2021-11-11T20:48:31.598 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-mds-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/32 2021-11-11T20:48:31.598 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-mgr-2:17.0.0-8977.ga3a8018b.el8.x86_64 3/32 2021-11-11T20:48:31.599 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-mon-2:17.0.0-8977.ga3a8018b.el8.x86_64 4/32 2021-11-11T20:48:31.599 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-osd-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/32 2021-11-11T20:48:31.600 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-mgr-modules-core-2:17.0.0-8977.ga3a8018b.el8. 6/32 2021-11-11T20:48:31.600 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2021-11-11T20:48:31.601 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2021-11-11T20:48:31.601 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2021-11-11T20:48:31.601 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2021-11-11T20:48:31.602 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2021-11-11T20:48:31.602 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2021-11-11T20:48:31.602 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2021-11-11T20:48:31.602 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2021-11-11T20:48:31.603 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2021-11-11T20:48:31.603 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2021-11-11T20:48:31.603 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2021-11-11T20:48:31.603 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2021-11-11T20:48:31.604 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2021-11-11T20:48:31.604 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2021-11-11T20:48:31.604 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 21/32 2021-11-11T20:48:31.604 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2021-11-11T20:48:31.604 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2021-11-11T20:48:31.605 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2021-11-11T20:48:31.605 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 25/32 2021-11-11T20:48:31.606 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 26/32 2021-11-11T20:48:31.606 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-cryptography-3.2.1-4.el8.x86_64 27/32 2021-11-11T20:48:31.607 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 28/32 2021-11-11T20:48:31.607 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 29/32 2021-11-11T20:48:31.607 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 30/32 2021-11-11T20:48:31.607 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2021-11-11T20:48:31.797 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cryptography-3.2.1-4.el8.x86_64 5/32 2021-11-11T20:48:31.985 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2021-11-11T20:48:32.128 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2021-11-11T20:48:32.292 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 32/32 2021-11-11T20:48:32.292 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:48:32.348 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2021-11-11T20:48:32.548 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 9/32 2021-11-11T20:48:32.834 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 10/32 2021-11-11T20:48:33.050 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2021-11-11T20:48:33.252 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2021-11-11T20:48:33.538 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2021-11-11T20:48:33.758 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-waitress-1.2.1-2.el8.1.noarch 14/32 2021-11-11T20:48:33.931 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2021-11-11T20:48:34.128 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2021-11-11T20:48:34.279 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2021-11-11T20:48:34.483 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2021-11-11T20:48:34.689 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2021-11-11T20:48:34.896 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2021-11-11T20:48:35.144 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2021-11-11T20:48:35.428 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2021-11-11T20:48:35.691 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2021-11-11T20:48:35.887 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2021-11-11T20:48:36.220 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2021-11-11T20:48:36.591 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2021-11-11T20:48:36.934 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-modules-core-2:17.0.0-8977.ga3a8018b.el8. 27/32 2021-11-11T20:48:37.156 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-2:17.0.0-8977.ga3a8018b.el8.x86_64 28/32 2021-11-11T20:48:38.634 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mgr-2:17.0.0-8977.ga3a8018b.el8.x86_64 28/32 2021-11-11T20:48:38.838 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-osd-2:17.0.0-8977.ga3a8018b.el8.x86_64 29/32 2021-11-11T20:48:39.228 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-osd-2:17.0.0-8977.ga3a8018b.el8.x86_64 29/32 2021-11-11T20:48:39.417 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mon-2:17.0.0-8977.ga3a8018b.el8.x86_64 30/32 2021-11-11T20:48:39.633 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mon-2:17.0.0-8977.ga3a8018b.el8.x86_64 30/32 2021-11-11T20:48:39.818 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mds-2:17.0.0-8977.ga3a8018b.el8.x86_64 31/32 2021-11-11T20:48:39.837 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mds-2:17.0.0-8977.ga3a8018b.el8.x86_64 31/32 2021-11-11T20:48:39.948 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-2:17.0.0-8977.ga3a8018b.el8.x86_64 32/32 2021-11-11T20:48:40.289 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-2:17.0.0-8977.ga3a8018b.el8.x86_64 32/32 2021-11-11T20:48:40.290 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/32 2021-11-11T20:48:40.291 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mds-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/32 2021-11-11T20:48:40.291 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-2:17.0.0-8977.ga3a8018b.el8.x86_64 3/32 2021-11-11T20:48:40.291 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mon-2:17.0.0-8977.ga3a8018b.el8.x86_64 4/32 2021-11-11T20:48:40.291 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-osd-2:17.0.0-8977.ga3a8018b.el8.x86_64 5/32 2021-11-11T20:48:40.292 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-modules-core-2:17.0.0-8977.ga3a8018b.el8. 6/32 2021-11-11T20:48:40.292 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2021-11-11T20:48:40.293 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2021-11-11T20:48:40.293 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2021-11-11T20:48:40.293 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2021-11-11T20:48:40.293 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2021-11-11T20:48:40.293 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2021-11-11T20:48:40.294 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2021-11-11T20:48:40.294 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2021-11-11T20:48:40.294 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2021-11-11T20:48:40.294 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2021-11-11T20:48:40.295 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2021-11-11T20:48:40.295 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2021-11-11T20:48:40.295 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2021-11-11T20:48:40.295 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2021-11-11T20:48:40.296 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 21/32 2021-11-11T20:48:40.296 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2021-11-11T20:48:40.296 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2021-11-11T20:48:40.296 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2021-11-11T20:48:40.296 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 25/32 2021-11-11T20:48:40.297 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 26/32 2021-11-11T20:48:40.297 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 27/32 2021-11-11T20:48:40.297 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2021-11-11T20:48:40.297 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2021-11-11T20:48:40.298 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 30/32 2021-11-11T20:48:40.298 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 31/32 2021-11-11T20:48:40.993 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cryptography-3.2.1-4.el8.x86_64 32/32 2021-11-11T20:48:40.993 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:48:44.274 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:47:54 PM UTC. 2021-11-11T20:48:47.044 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:48:47.045 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:48:47.045 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:47.045 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mds-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:47.046 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:47.046 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr-modules-core-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:48:47.046 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mon-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:47.046 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-osd-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:47.047 INFO:teuthology.orchestra.run.smithi198.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2021-11-11T20:48:47.047 INFO:teuthology.orchestra.run.smithi198.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2021-11-11T20:48:47.047 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cffi-1.11.5-5.el8.x86_64 2021-11-11T20:48:47.047 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cheroot-8.5.2-1.el8.noarch 2021-11-11T20:48:47.047 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2021-11-11T20:48:47.048 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cryptography-3.2.1-4.el8.x86_64 2021-11-11T20:48:47.048 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jaraco-6.2-6.el8.noarch 2021-11-11T20:48:47.048 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2021-11-11T20:48:47.048 INFO:teuthology.orchestra.run.smithi198.stdout: python3-logutils-0.3.5-11.el8.noarch 2021-11-11T20:48:47.049 INFO:teuthology.orchestra.run.smithi198.stdout: python3-mako-1.0.6-13.el8.noarch 2021-11-11T20:48:47.049 INFO:teuthology.orchestra.run.smithi198.stdout: python3-markupsafe-0.23-19.el8.x86_64 2021-11-11T20:48:47.049 INFO:teuthology.orchestra.run.smithi198.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2021-11-11T20:48:47.049 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pecan-1.3.2-9.el8.noarch 2021-11-11T20:48:47.049 INFO:teuthology.orchestra.run.smithi198.stdout: python3-portend-2.6-1.el8.noarch 2021-11-11T20:48:47.050 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2021-11-11T20:48:47.050 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pycparser-2.14-14.el8.noarch 2021-11-11T20:48:47.050 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pytz-2017.2-9.el8.noarch 2021-11-11T20:48:47.051 INFO:teuthology.orchestra.run.smithi198.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2021-11-11T20:48:47.051 INFO:teuthology.orchestra.run.smithi198.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2021-11-11T20:48:47.052 INFO:teuthology.orchestra.run.smithi198.stdout: python3-tempora-1.14.1-5.el8.noarch 2021-11-11T20:48:47.052 INFO:teuthology.orchestra.run.smithi198.stdout: python3-trustme-0.6.0-4.el8.noarch 2021-11-11T20:48:47.052 INFO:teuthology.orchestra.run.smithi198.stdout: python3-waitress-1.2.1-2.el8.1.noarch 2021-11-11T20:48:47.052 INFO:teuthology.orchestra.run.smithi198.stdout: python3-webob-1.8.5-1.el8.1.noarch 2021-11-11T20:48:47.053 INFO:teuthology.orchestra.run.smithi198.stdout: python3-webtest-2.0.33-1.el8.noarch 2021-11-11T20:48:47.053 INFO:teuthology.orchestra.run.smithi198.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2021-11-11T20:48:47.053 INFO:teuthology.orchestra.run.smithi198.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2021-11-11T20:48:47.053 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:48:47.054 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:48:47.188 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph-base 2021-11-11T20:48:47.516 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:48:48.045 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 80 kB/s | 2.4 kB 00:00 2021-11-11T20:48:52.878 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:48:03 PM UTC. 2021-11-11T20:48:53.340 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 98 kB/s | 2.8 kB 00:00 2021-11-11T20:48:55.659 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:48:55.659 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:48:55.660 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:55.660 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mds-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:55.660 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:55.660 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-modules-core-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:48:55.661 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mon-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:55.661 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-osd-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:48:55.661 INFO:teuthology.orchestra.run.smithi096.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2021-11-11T20:48:55.661 INFO:teuthology.orchestra.run.smithi096.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2021-11-11T20:48:55.661 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cffi-1.11.5-5.el8.x86_64 2021-11-11T20:48:55.662 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cheroot-8.5.2-1.el8.noarch 2021-11-11T20:48:55.662 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2021-11-11T20:48:55.662 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cryptography-3.2.1-4.el8.x86_64 2021-11-11T20:48:55.662 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-6.2-6.el8.noarch 2021-11-11T20:48:55.663 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2021-11-11T20:48:55.663 INFO:teuthology.orchestra.run.smithi096.stdout: python3-logutils-0.3.5-11.el8.noarch 2021-11-11T20:48:55.663 INFO:teuthology.orchestra.run.smithi096.stdout: python3-mako-1.0.6-13.el8.noarch 2021-11-11T20:48:55.663 INFO:teuthology.orchestra.run.smithi096.stdout: python3-markupsafe-0.23-19.el8.x86_64 2021-11-11T20:48:55.664 INFO:teuthology.orchestra.run.smithi096.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2021-11-11T20:48:55.664 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pecan-1.3.2-9.el8.noarch 2021-11-11T20:48:55.664 INFO:teuthology.orchestra.run.smithi096.stdout: python3-portend-2.6-1.el8.noarch 2021-11-11T20:48:55.664 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2021-11-11T20:48:55.665 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pycparser-2.14-14.el8.noarch 2021-11-11T20:48:55.665 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pytz-2017.2-9.el8.noarch 2021-11-11T20:48:55.665 INFO:teuthology.orchestra.run.smithi096.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2021-11-11T20:48:55.665 INFO:teuthology.orchestra.run.smithi096.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2021-11-11T20:48:55.665 INFO:teuthology.orchestra.run.smithi096.stdout: python3-tempora-1.14.1-5.el8.noarch 2021-11-11T20:48:55.666 INFO:teuthology.orchestra.run.smithi096.stdout: python3-trustme-0.6.0-4.el8.noarch 2021-11-11T20:48:55.666 INFO:teuthology.orchestra.run.smithi096.stdout: python3-waitress-1.2.1-2.el8.1.noarch 2021-11-11T20:48:55.666 INFO:teuthology.orchestra.run.smithi096.stdout: python3-webob-1.8.5-1.el8.1.noarch 2021-11-11T20:48:55.666 INFO:teuthology.orchestra.run.smithi096.stdout: python3-webtest-2.0.33-1.el8.noarch 2021-11-11T20:48:55.667 INFO:teuthology.orchestra.run.smithi096.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2021-11-11T20:48:55.667 INFO:teuthology.orchestra.run.smithi096.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2021-11-11T20:48:55.667 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:48:55.667 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:48:55.788 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-base 2021-11-11T20:48:56.119 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:48:56.608 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 97 kB/s | 2.8 kB 00:00 2021-11-11T20:48:58.408 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:49:00.547 INFO:teuthology.orchestra.run.smithi198.stdout:Package ceph-base-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:49:00.994 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:49:00.995 INFO:teuthology.orchestra.run.smithi198.stdout:Nothing to do. 2021-11-11T20:49:00.996 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:49:01.093 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install cephadm 2021-11-11T20:49:01.420 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:49:01.619 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 78 kB/s | 2.4 kB 00:00 2021-11-11T20:49:01.969 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 78 kB/s | 2.4 kB 00:00 2021-11-11T20:49:02.098 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 97 kB/s | 2.8 kB 00:00 2021-11-11T20:49:04.039 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:49:04.040 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:49:04.040 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repository Size 2021-11-11T20:49:04.041 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:49:04.041 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:49:04.041 INFO:teuthology.orchestra.run.smithi198.stdout: cephadm noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 68 k 2021-11-11T20:49:04.041 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:49:04.042 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:49:04.042 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:49:04.042 INFO:teuthology.orchestra.run.smithi198.stdout:Install 1 Package 2021-11-11T20:49:04.042 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:49:04.043 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 68 k 2021-11-11T20:49:04.043 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 305 k 2021-11-11T20:49:04.043 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:49:04.568 INFO:teuthology.orchestra.run.smithi198.stdout:cephadm-17.0.0-8977.ga3a8018b.el8.noarch.rpm 130 kB/s | 68 kB 00:00 2021-11-11T20:49:04.569 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:49:04.569 INFO:teuthology.orchestra.run.smithi198.stdout:Total 129 kB/s | 68 kB 00:00 2021-11-11T20:49:04.569 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:49:04.577 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:49:04.577 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:49:04.579 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:49:04.583 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:49:04.659 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:49:06.968 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:49:07.075 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:49:07.200 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:49:07.363 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:49:07.610 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:49:07.611 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:49:09.239 INFO:teuthology.orchestra.run.smithi096.stdout:Package ceph-base-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:49:09.676 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:49:09.677 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2021-11-11T20:49:09.677 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:49:09.785 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install cephadm 2021-11-11T20:49:10.109 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:49:10.594 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 92 kB/s | 2.8 kB 00:00 2021-11-11T20:49:10.774 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 76 kB/s | 2.4 kB 00:00 2021-11-11T20:49:12.628 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:49:12.630 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:49:12.630 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2021-11-11T20:49:12.630 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:49:12.631 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:49:12.631 INFO:teuthology.orchestra.run.smithi096.stdout: cephadm noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 68 k 2021-11-11T20:49:12.631 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:49:12.631 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:49:12.632 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:49:12.632 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2021-11-11T20:49:12.632 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:49:12.632 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 68 k 2021-11-11T20:49:12.633 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 305 k 2021-11-11T20:49:12.633 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:49:12.860 INFO:teuthology.orchestra.run.smithi096.stdout:cephadm-17.0.0-8977.ga3a8018b.el8.noarch.rpm 298 kB/s | 68 kB 00:00 2021-11-11T20:49:12.860 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:49:12.861 INFO:teuthology.orchestra.run.smithi096.stdout:Total 296 kB/s | 68 kB 00:00 2021-11-11T20:49:12.861 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:49:12.868 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:49:12.869 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:49:12.870 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:49:12.875 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:49:12.916 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:49:15.225 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:49:15.434 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:49:15.594 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:49:15.797 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:49:15.798 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:49:19.345 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:48:38 PM UTC. 2021-11-11T20:49:22.095 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:49:22.096 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:49:22.096 INFO:teuthology.orchestra.run.smithi198.stdout: cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:49:22.096 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:49:22.096 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:49:22.248 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph-immutable-object-cache 2021-11-11T20:49:22.587 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:49:23.109 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 75 kB/s | 2.4 kB 00:00 2021-11-11T20:49:27.614 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:48:47 PM UTC. 2021-11-11T20:49:28.443 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 93 kB/s | 2.8 kB 00:00 2021-11-11T20:49:30.322 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:49:30.322 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:49:30.322 INFO:teuthology.orchestra.run.smithi096.stdout: cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:49:30.323 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:49:30.323 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:49:30.438 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-immutable-object-cache 2021-11-11T20:49:30.774 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:49:31.276 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 94 kB/s | 2.8 kB 00:00 2021-11-11T20:49:33.443 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:49:36.086 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:49:36.088 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:49:36.088 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repo Size 2021-11-11T20:49:36.088 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:49:36.089 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:49:36.090 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-immutable-object-cache x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 169 k 2021-11-11T20:49:36.090 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:49:36.090 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:49:36.091 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:49:36.091 INFO:teuthology.orchestra.run.smithi198.stdout:Install 1 Package 2021-11-11T20:49:36.091 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:49:36.092 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 169 k 2021-11-11T20:49:36.092 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 567 k 2021-11-11T20:49:36.092 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:49:36.314 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 79 kB/s | 2.4 kB 00:00 2021-11-11T20:49:36.331 INFO:teuthology.orchestra.run.smithi198.stdout:ceph-immutable-object-cache-17.0.0-8977.ga3a801 696 kB/s | 169 kB 00:00 2021-11-11T20:49:36.332 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:49:36.332 INFO:teuthology.orchestra.run.smithi198.stdout:Total 692 kB/s | 169 kB 00:00 2021-11-11T20:49:36.333 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:49:36.346 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:49:36.347 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:49:36.398 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:49:36.402 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:49:36.517 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:49:36.696 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-immutable-object-cache-2:17.0.0-8977.ga3a8018b. 1/1 2021-11-11T20:49:37.052 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.0.0-8977.ga3a8018b. 1/1 2021-11-11T20:49:37.289 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-immutable-object-cache-2:17.0.0-8977.ga3a8018b. 1/1 2021-11-11T20:49:37.291 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:49:41.800 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:49:44.415 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:49:44.416 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:49:44.416 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repo Size 2021-11-11T20:49:44.416 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:49:44.416 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:49:44.417 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-immutable-object-cache x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 169 k 2021-11-11T20:49:44.417 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:49:44.417 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:49:44.417 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:49:44.418 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2021-11-11T20:49:44.418 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:49:44.418 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 169 k 2021-11-11T20:49:44.418 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 567 k 2021-11-11T20:49:44.419 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:49:44.702 INFO:teuthology.orchestra.run.smithi096.stdout:ceph-immutable-object-cache-17.0.0-8977.ga3a801 592 kB/s | 169 kB 00:00 2021-11-11T20:49:44.702 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:49:44.703 INFO:teuthology.orchestra.run.smithi096.stdout:Total 589 kB/s | 169 kB 00:00 2021-11-11T20:49:44.703 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:49:44.716 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:49:44.716 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:49:44.769 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:49:44.773 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:49:44.898 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:49:45.061 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-immutable-object-cache-2:17.0.0-8977.ga3a8018b. 1/1 2021-11-11T20:49:45.406 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.0.0-8977.ga3a8018b. 1/1 2021-11-11T20:49:45.634 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-immutable-object-cache-2:17.0.0-8977.ga3a8018b. 1/1 2021-11-11T20:49:45.634 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:49:49.065 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:49:13 PM UTC. 2021-11-11T20:49:51.848 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:49:51.848 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:49:51.848 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-immutable-object-cache-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:49:51.849 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:49:51.849 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:49:51.985 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph-mgr 2021-11-11T20:49:52.318 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:49:52.855 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 79 kB/s | 2.4 kB 00:00 2021-11-11T20:49:57.424 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:49:21 PM UTC. 2021-11-11T20:49:58.185 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 94 kB/s | 2.8 kB 00:00 2021-11-11T20:50:00.151 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:50:00.151 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:50:00.151 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-immutable-object-cache-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:50:00.152 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:50:00.152 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:50:00.289 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-mgr 2021-11-11T20:50:00.620 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:50:01.107 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 90 kB/s | 2.8 kB 00:00 2021-11-11T20:50:03.430 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:50:05.645 INFO:teuthology.orchestra.run.smithi198.stdout:Package ceph-mgr-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:50:06.065 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 83 kB/s | 2.4 kB 00:00 2021-11-11T20:50:06.106 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:50:06.107 INFO:teuthology.orchestra.run.smithi198.stdout:Nothing to do. 2021-11-11T20:50:06.107 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:50:06.214 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph-mgr-dashboard 2021-11-11T20:50:06.540 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:50:07.076 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 77 kB/s | 2.4 kB 00:00 2021-11-11T20:50:07.208 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 89 kB/s | 2.8 kB 00:00 2021-11-11T20:50:09.057 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:50:09.058 INFO:teuthology.orchestra.run.smithi198.stdout:=================================================================================================== 2021-11-11T20:50:09.058 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repository Size 2021-11-11T20:50:09.059 INFO:teuthology.orchestra.run.smithi198.stdout:=================================================================================================== 2021-11-11T20:50:09.059 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:50:09.059 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr-dashboard noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 1.5 M 2021-11-11T20:50:09.060 INFO:teuthology.orchestra.run.smithi198.stdout:Installing dependencies: 2021-11-11T20:50:09.060 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-grafana-dashboards noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 23 k 2021-11-11T20:50:09.060 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-prometheus-alerts noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 13 k 2021-11-11T20:50:09.061 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2021-11-11T20:50:09.061 INFO:teuthology.orchestra.run.smithi198.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2021-11-11T20:50:09.061 INFO:teuthology.orchestra.run.smithi198.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2021-11-11T20:50:09.062 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:50:09.062 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:50:09.062 INFO:teuthology.orchestra.run.smithi198.stdout:=================================================================================================== 2021-11-11T20:50:09.062 INFO:teuthology.orchestra.run.smithi198.stdout:Install 6 Packages 2021-11-11T20:50:09.062 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:50:09.063 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 1.8 M 2021-11-11T20:50:09.064 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 7.5 M 2021-11-11T20:50:09.064 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:50:09.208 INFO:teuthology.orchestra.run.smithi198.stdout:(1/6): ceph-prometheus-alerts-17.0.0-8977.ga3a8 90 kB/s | 13 kB 00:00 2021-11-11T20:50:09.245 INFO:teuthology.orchestra.run.smithi198.stdout:(2/6): ceph-grafana-dashboards-17.0.0-8977.ga3a 125 kB/s | 23 kB 00:00 2021-11-11T20:50:09.284 INFO:teuthology.orchestra.run.smithi198.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 442 kB/s | 34 kB 00:00 2021-11-11T20:50:09.313 INFO:teuthology.orchestra.run.smithi198.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 1.4 MB/s | 43 kB 00:00 2021-11-11T20:50:09.368 INFO:teuthology.orchestra.run.smithi198.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 1.6 MB/s | 196 kB 00:00 2021-11-11T20:50:09.544 INFO:teuthology.orchestra.run.smithi198.stdout:(6/6): ceph-mgr-dashboard-17.0.0-8977.ga3a8018b 3.2 MB/s | 1.5 MB 00:00 2021-11-11T20:50:09.546 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:50:09.547 INFO:teuthology.orchestra.run.smithi198.stdout:Total 3.8 MB/s | 1.8 MB 00:00 2021-11-11T20:50:09.549 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:50:09.558 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:50:09.559 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:50:09.613 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:50:09.621 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:50:09.855 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:50:10.067 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2021-11-11T20:50:10.298 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2021-11-11T20:50:10.575 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2021-11-11T20:50:10.714 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-prometheus-alerts-2:17.0.0-8977.ga3a8018b.el8.n 4/6 2021-11-11T20:50:11.040 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-grafana-dashboards-2:17.0.0-8977.ga3a8018b.el8. 5/6 2021-11-11T20:50:11.252 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-mgr-dashboard-2:17.0.0-8977.ga3a8018b.el8.noarc 6/6 2021-11-11T20:50:11.477 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.0.0-8977.ga3a8018b.el8.noarc 6/6 2021-11-11T20:50:11.477 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-grafana-dashboards-2:17.0.0-8977.ga3a8018b.el8. 1/6 2021-11-11T20:50:11.477 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-mgr-dashboard-2:17.0.0-8977.ga3a8018b.el8.noarc 2/6 2021-11-11T20:50:11.478 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-prometheus-alerts-2:17.0.0-8977.ga3a8018b.el8.n 3/6 2021-11-11T20:50:11.478 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2021-11-11T20:50:11.478 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2021-11-11T20:50:11.651 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:04 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:50:11.728 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2021-11-11T20:50:11.728 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:50:13.782 INFO:teuthology.orchestra.run.smithi096.stdout:Package ceph-mgr-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:50:14.217 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:50:14.219 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2021-11-11T20:50:14.219 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:50:14.330 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-mgr-dashboard 2021-11-11T20:50:14.657 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:50:15.164 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 96 kB/s | 2.8 kB 00:00 2021-11-11T20:50:15.341 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 81 kB/s | 2.4 kB 00:00 2021-11-11T20:50:17.234 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:50:17.235 INFO:teuthology.orchestra.run.smithi096.stdout:=================================================================================================== 2021-11-11T20:50:17.236 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2021-11-11T20:50:17.236 INFO:teuthology.orchestra.run.smithi096.stdout:=================================================================================================== 2021-11-11T20:50:17.237 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:50:17.237 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-dashboard noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 1.5 M 2021-11-11T20:50:17.238 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2021-11-11T20:50:17.238 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-grafana-dashboards noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 23 k 2021-11-11T20:50:17.239 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-prometheus-alerts noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 13 k 2021-11-11T20:50:17.239 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2021-11-11T20:50:17.239 INFO:teuthology.orchestra.run.smithi096.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2021-11-11T20:50:17.239 INFO:teuthology.orchestra.run.smithi096.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2021-11-11T20:50:17.240 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:50:17.240 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:50:17.240 INFO:teuthology.orchestra.run.smithi096.stdout:=================================================================================================== 2021-11-11T20:50:17.240 INFO:teuthology.orchestra.run.smithi096.stdout:Install 6 Packages 2021-11-11T20:50:17.240 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:50:17.241 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 1.8 M 2021-11-11T20:50:17.241 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 7.5 M 2021-11-11T20:50:17.241 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:50:17.394 INFO:teuthology.orchestra.run.smithi096.stdout:(1/6): ceph-prometheus-alerts-17.0.0-8977.ga3a8 84 kB/s | 13 kB 00:00 2021-11-11T20:50:17.412 INFO:teuthology.orchestra.run.smithi096.stdout:(2/6): ceph-grafana-dashboards-17.0.0-8977.ga3a 131 kB/s | 23 kB 00:00 2021-11-11T20:50:17.468 INFO:teuthology.orchestra.run.smithi096.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 452 kB/s | 34 kB 00:00 2021-11-11T20:50:17.499 INFO:teuthology.orchestra.run.smithi096.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 1.4 MB/s | 43 kB 00:00 2021-11-11T20:50:17.554 INFO:teuthology.orchestra.run.smithi096.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 1.4 MB/s | 196 kB 00:00 2021-11-11T20:50:17.623 INFO:teuthology.orchestra.run.smithi096.stdout:(6/6): ceph-mgr-dashboard-17.0.0-8977.ga3a8018b 4.0 MB/s | 1.5 MB 00:00 2021-11-11T20:50:17.625 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:50:17.626 INFO:teuthology.orchestra.run.smithi096.stdout:Total 4.7 MB/s | 1.8 MB 00:00 2021-11-11T20:50:17.628 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:50:17.636 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:50:17.637 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:50:17.688 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:50:17.692 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:50:17.984 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:50:18.198 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2021-11-11T20:50:18.379 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2021-11-11T20:50:18.631 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2021-11-11T20:50:18.777 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-prometheus-alerts-2:17.0.0-8977.ga3a8018b.el8.n 4/6 2021-11-11T20:50:19.115 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-grafana-dashboards-2:17.0.0-8977.ga3a8018b.el8. 5/6 2021-11-11T20:50:19.299 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-dashboard-2:17.0.0-8977.ga3a8018b.el8.noarc 6/6 2021-11-11T20:50:19.531 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.0.0-8977.ga3a8018b.el8.noarc 6/6 2021-11-11T20:50:19.531 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-grafana-dashboards-2:17.0.0-8977.ga3a8018b.el8. 1/6 2021-11-11T20:50:19.532 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-dashboard-2:17.0.0-8977.ga3a8018b.el8.noarc 2/6 2021-11-11T20:50:19.532 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-prometheus-alerts-2:17.0.0-8977.ga3a8018b.el8.n 3/6 2021-11-11T20:50:19.533 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2021-11-11T20:50:19.533 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2021-11-11T20:50:19.792 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2021-11-11T20:50:19.792 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:50:23.474 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:49:43 PM UTC. 2021-11-11T20:50:26.169 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:50:26.169 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:50:26.170 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-grafana-dashboards-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:50:26.170 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr-dashboard-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:50:26.170 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-prometheus-alerts-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:50:26.170 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jwt-1.6.1-2.el8.noarch 2021-11-11T20:50:26.171 INFO:teuthology.orchestra.run.smithi198.stdout: python3-repoze-lru-0.7-6.el8.noarch 2021-11-11T20:50:26.176 INFO:teuthology.orchestra.run.smithi198.stdout: python3-routes-2.4.1-12.el8.noarch 2021-11-11T20:50:26.176 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:50:26.177 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:50:26.323 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph-mgr-diskprediction-local 2021-11-11T20:50:26.655 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:50:27.182 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 81 kB/s | 2.4 kB 00:00 2021-11-11T20:50:31.640 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:49:51 PM UTC. 2021-11-11T20:50:32.479 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 95 kB/s | 2.8 kB 00:00 2021-11-11T20:50:34.384 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:50:34.385 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:50:34.385 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-grafana-dashboards-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:50:34.385 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-dashboard-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:50:34.385 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-prometheus-alerts-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:50:34.386 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jwt-1.6.1-2.el8.noarch 2021-11-11T20:50:34.386 INFO:teuthology.orchestra.run.smithi096.stdout: python3-repoze-lru-0.7-6.el8.noarch 2021-11-11T20:50:34.386 INFO:teuthology.orchestra.run.smithi096.stdout: python3-routes-2.4.1-12.el8.noarch 2021-11-11T20:50:34.387 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:50:34.387 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:50:34.516 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-mgr-diskprediction-local 2021-11-11T20:50:34.845 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:50:35.353 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 85 kB/s | 2.8 kB 00:00 2021-11-11T20:50:37.472 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:50:40.093 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:50:40.093 INFO:teuthology.orchestra.run.smithi198.stdout:====================================================================================================================== 2021-11-11T20:50:40.094 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repository Size 2021-11-11T20:50:40.096 INFO:teuthology.orchestra.run.smithi198.stdout:====================================================================================================================== 2021-11-11T20:50:40.096 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:50:40.097 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr-diskprediction-local noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 7.4 M 2021-11-11T20:50:40.097 INFO:teuthology.orchestra.run.smithi198.stdout:Installing dependencies: 2021-11-11T20:50:40.098 INFO:teuthology.orchestra.run.smithi198.stdout: libgfortran x86_64 8.4.1-1.el8 rhel-8-for-x86_64-baseos-rpms 639 k 2021-11-11T20:50:40.098 INFO:teuthology.orchestra.run.smithi198.stdout: libquadmath x86_64 8.4.1-1.el8 rhel-8-for-x86_64-baseos-rpms 169 k 2021-11-11T20:50:40.099 INFO:teuthology.orchestra.run.smithi198.stdout: openblas x86_64 0.3.12-1.el8 rhel-8-for-x86_64-appstream-rpms 4.6 M 2021-11-11T20:50:40.100 INFO:teuthology.orchestra.run.smithi198.stdout: openblas-threads x86_64 0.3.12-1.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2021-11-11T20:50:40.100 INFO:teuthology.orchestra.run.smithi198.stdout: python3-numpy x86_64 1:1.14.3-9.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2021-11-11T20:50:40.101 INFO:teuthology.orchestra.run.smithi198.stdout: python3-numpy-f2py x86_64 1:1.14.3-9.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2021-11-11T20:50:40.101 INFO:teuthology.orchestra.run.smithi198.stdout: python3-scipy x86_64 1.0.0-20.module+el8.1.0+3334+5cb623d7 rhel-8-for-x86_64-appstream-rpms 15 M 2021-11-11T20:50:40.102 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:50:40.102 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:50:40.103 INFO:teuthology.orchestra.run.smithi198.stdout:====================================================================================================================== 2021-11-11T20:50:40.104 INFO:teuthology.orchestra.run.smithi198.stdout:Install 8 Packages 2021-11-11T20:50:40.104 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:50:40.105 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 36 M 2021-11-11T20:50:40.105 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 199 M 2021-11-11T20:50:40.105 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:50:40.132 INFO:teuthology.orchestra.run.smithi198.stdout:(1/8): libquadmath-8.4.1-1.el8.x86_64.rpm 4.2 MB/s | 169 kB 00:00 2021-11-11T20:50:40.150 INFO:teuthology.orchestra.run.smithi198.stdout:(2/8): libgfortran-8.4.1-1.el8.x86_64.rpm 11 MB/s | 639 kB 00:00 2021-11-11T20:50:40.184 INFO:teuthology.orchestra.run.smithi198.stdout:(3/8): python3-numpy-1.14.3-9.el8.x86_64.rpm 71 MB/s | 3.7 MB 00:00 2021-11-11T20:50:40.241 INFO:teuthology.orchestra.run.smithi198.stdout:(4/8): openblas-threads-0.3.12-1.el8.x86_64.rpm 84 MB/s | 4.7 MB 00:00 2021-11-11T20:50:40.320 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 85 kB/s | 2.4 kB 00:00 2021-11-11T20:50:40.383 INFO:teuthology.orchestra.run.smithi198.stdout:(5/8): python3-scipy-1.0.0-20.module+el8.1.0+33 63 MB/s | 15 MB 00:00 2021-11-11T20:50:40.413 INFO:teuthology.orchestra.run.smithi198.stdout:(6/8): openblas-0.3.12-1.el8.x86_64.rpm 27 MB/s | 4.6 MB 00:00 2021-11-11T20:50:40.415 INFO:teuthology.orchestra.run.smithi198.stdout:(7/8): python3-numpy-f2py-1.14.3-9.el8.x86_64.r 7.0 MB/s | 225 kB 00:00 2021-11-11T20:50:41.514 INFO:teuthology.orchestra.run.smithi198.stdout:(8/8): ceph-mgr-diskprediction-local-17.0.0-897 5.2 MB/s | 7.4 MB 00:01 2021-11-11T20:50:41.515 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:50:41.516 INFO:teuthology.orchestra.run.smithi198.stdout:Total 25 MB/s | 36 MB 00:01 2021-11-11T20:50:41.531 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:50:41.550 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:50:41.551 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:50:41.851 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:50:41.855 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:50:42.442 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:50:42.642 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : libquadmath-8.4.1-1.el8.x86_64 1/8 2021-11-11T20:50:42.784 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: libquadmath-8.4.1-1.el8.x86_64 1/8 2021-11-11T20:50:42.975 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : libgfortran-8.4.1-1.el8.x86_64 2/8 2021-11-11T20:50:43.519 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: libgfortran-8.4.1-1.el8.x86_64 2/8 2021-11-11T20:50:43.717 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : openblas-threads-0.3.12-1.el8.x86_64 3/8 2021-11-11T20:50:44.244 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: openblas-threads-0.3.12-1.el8.x86_64 3/8 2021-11-11T20:50:44.442 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : openblas-0.3.12-1.el8.x86_64 4/8 2021-11-11T20:50:45.176 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: openblas-0.3.12-1.el8.x86_64 4/8 2021-11-11T20:50:45.509 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-numpy-1:1.14.3-9.el8.x86_64 5/8 2021-11-11T20:50:45.766 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-numpy-f2py-1:1.14.3-9.el8.x86_64 6/8 2021-11-11T20:50:45.809 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:50:47.769 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-9.el8.x86_64 6/8 2021-11-11T20:50:48.441 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:50:48.442 INFO:teuthology.orchestra.run.smithi096.stdout:====================================================================================================================== 2021-11-11T20:50:48.443 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2021-11-11T20:50:48.443 INFO:teuthology.orchestra.run.smithi096.stdout:====================================================================================================================== 2021-11-11T20:50:48.444 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:50:48.444 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-diskprediction-local noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 7.4 M 2021-11-11T20:50:48.445 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2021-11-11T20:50:48.445 INFO:teuthology.orchestra.run.smithi096.stdout: libgfortran x86_64 8.4.1-1.el8 rhel-8-for-x86_64-baseos-rpms 639 k 2021-11-11T20:50:48.446 INFO:teuthology.orchestra.run.smithi096.stdout: libquadmath x86_64 8.4.1-1.el8 rhel-8-for-x86_64-baseos-rpms 169 k 2021-11-11T20:50:48.447 INFO:teuthology.orchestra.run.smithi096.stdout: openblas x86_64 0.3.12-1.el8 rhel-8-for-x86_64-appstream-rpms 4.6 M 2021-11-11T20:50:48.447 INFO:teuthology.orchestra.run.smithi096.stdout: openblas-threads x86_64 0.3.12-1.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2021-11-11T20:50:48.448 INFO:teuthology.orchestra.run.smithi096.stdout: python3-numpy x86_64 1:1.14.3-9.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2021-11-11T20:50:48.448 INFO:teuthology.orchestra.run.smithi096.stdout: python3-numpy-f2py x86_64 1:1.14.3-9.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2021-11-11T20:50:48.449 INFO:teuthology.orchestra.run.smithi096.stdout: python3-scipy x86_64 1.0.0-20.module+el8.1.0+3334+5cb623d7 rhel-8-for-x86_64-appstream-rpms 15 M 2021-11-11T20:50:48.450 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:50:48.450 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:50:48.452 INFO:teuthology.orchestra.run.smithi096.stdout:====================================================================================================================== 2021-11-11T20:50:48.453 INFO:teuthology.orchestra.run.smithi096.stdout:Install 8 Packages 2021-11-11T20:50:48.453 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:50:48.455 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 36 M 2021-11-11T20:50:48.455 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 199 M 2021-11-11T20:50:48.456 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:50:48.514 INFO:teuthology.orchestra.run.smithi096.stdout:(1/8): python3-numpy-1.14.3-9.el8.x86_64.rpm 54 MB/s | 3.7 MB 00:00 2021-11-11T20:50:48.572 INFO:teuthology.orchestra.run.smithi096.stdout:(2/8): openblas-threads-0.3.12-1.el8.x86_64.rpm 83 MB/s | 4.7 MB 00:00 2021-11-11T20:50:48.722 INFO:teuthology.orchestra.run.smithi096.stdout:(3/8): python3-scipy-1.0.0-20.module+el8.1.0+33 52 MB/s | 15 MB 00:00 2021-11-11T20:50:48.744 INFO:teuthology.orchestra.run.smithi096.stdout:(4/8): python3-numpy-f2py-1.14.3-9.el8.x86_64.r 10 MB/s | 225 kB 00:00 2021-11-11T20:50:48.757 INFO:teuthology.orchestra.run.smithi096.stdout:(5/8): openblas-0.3.12-1.el8.x86_64.rpm 25 MB/s | 4.6 MB 00:00 2021-11-11T20:50:48.780 INFO:teuthology.orchestra.run.smithi096.stdout:(6/8): libquadmath-8.4.1-1.el8.x86_64.rpm 4.6 MB/s | 169 kB 00:00 2021-11-11T20:50:48.787 INFO:teuthology.orchestra.run.smithi096.stdout:(7/8): libgfortran-8.4.1-1.el8.x86_64.rpm 24 MB/s | 639 kB 00:00 2021-11-11T20:50:49.035 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-scipy-1.0.0-20.module+el8.1.0+3334+5cb623d7. 7/8 2021-11-11T20:50:49.225 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-mgr-diskprediction-local-2:17.0.0-8977.ga3a8018 8/8 2021-11-11T20:50:49.465 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.0.0-8977.ga3a8018 8/8 2021-11-11T20:50:49.465 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.0.0-8977.ga3a8018 1/8 2021-11-11T20:50:49.465 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : libquadmath-8.4.1-1.el8.x86_64 2/8 2021-11-11T20:50:49.465 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : libgfortran-8.4.1-1.el8.x86_64 3/8 2021-11-11T20:50:49.466 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-numpy-1:1.14.3-9.el8.x86_64 4/8 2021-11-11T20:50:49.466 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-scipy-1.0.0-20.module+el8.1.0+3334+5cb623d7. 5/8 2021-11-11T20:50:49.466 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : openblas-threads-0.3.12-1.el8.x86_64 6/8 2021-11-11T20:50:49.466 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : openblas-0.3.12-1.el8.x86_64 7/8 2021-11-11T20:50:49.735 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-numpy-f2py-1:1.14.3-9.el8.x86_64 8/8 2021-11-11T20:50:49.735 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:50:50.133 INFO:teuthology.orchestra.run.smithi096.stdout:(8/8): ceph-mgr-diskprediction-local-17.0.0-897 4.4 MB/s | 7.4 MB 00:01 2021-11-11T20:50:50.135 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:50:50.136 INFO:teuthology.orchestra.run.smithi096.stdout:Total 21 MB/s | 36 MB 00:01 2021-11-11T20:50:50.151 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:50:50.172 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:50:50.172 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:50:50.467 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:50:50.471 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:50:51.028 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:50:51.197 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libquadmath-8.4.1-1.el8.x86_64 1/8 2021-11-11T20:50:51.341 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: libquadmath-8.4.1-1.el8.x86_64 1/8 2021-11-11T20:50:51.579 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libgfortran-8.4.1-1.el8.x86_64 2/8 2021-11-11T20:50:52.153 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: libgfortran-8.4.1-1.el8.x86_64 2/8 2021-11-11T20:50:52.372 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : openblas-threads-0.3.12-1.el8.x86_64 3/8 2021-11-11T20:50:52.902 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: openblas-threads-0.3.12-1.el8.x86_64 3/8 2021-11-11T20:50:53.078 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : openblas-0.3.12-1.el8.x86_64 4/8 2021-11-11T20:50:53.805 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: openblas-0.3.12-1.el8.x86_64 4/8 2021-11-11T20:50:54.154 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-numpy-1:1.14.3-9.el8.x86_64 5/8 2021-11-11T20:50:54.411 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-numpy-f2py-1:1.14.3-9.el8.x86_64 6/8 2021-11-11T20:50:56.423 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-9.el8.x86_64 6/8 2021-11-11T20:50:57.636 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-scipy-1.0.0-20.module+el8.1.0+3334+5cb623d7. 7/8 2021-11-11T20:50:57.794 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-diskprediction-local-2:17.0.0-8977.ga3a8018 8/8 2021-11-11T20:50:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.0.0-8977.ga3a8018 8/8 2021-11-11T20:50:58.079 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.0.0-8977.ga3a8018 1/8 2021-11-11T20:50:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-numpy-1:1.14.3-9.el8.x86_64 2/8 2021-11-11T20:50:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-scipy-1.0.0-20.module+el8.1.0+3334+5cb623d7. 3/8 2021-11-11T20:50:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : openblas-threads-0.3.12-1.el8.x86_64 4/8 2021-11-11T20:50:58.080 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : openblas-0.3.12-1.el8.x86_64 5/8 2021-11-11T20:50:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-numpy-f2py-1:1.14.3-9.el8.x86_64 6/8 2021-11-11T20:50:58.081 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libquadmath-8.4.1-1.el8.x86_64 7/8 2021-11-11T20:50:58.346 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libgfortran-8.4.1-1.el8.x86_64 8/8 2021-11-11T20:50:58.346 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:51:01.569 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:50:17 PM UTC. 2021-11-11T20:51:04.317 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:51:04.317 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:51:04.317 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr-diskprediction-local-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:51:04.318 INFO:teuthology.orchestra.run.smithi198.stdout: libgfortran-8.4.1-1.el8.x86_64 2021-11-11T20:51:04.318 INFO:teuthology.orchestra.run.smithi198.stdout: libquadmath-8.4.1-1.el8.x86_64 2021-11-11T20:51:04.318 INFO:teuthology.orchestra.run.smithi198.stdout: openblas-0.3.12-1.el8.x86_64 2021-11-11T20:51:04.318 INFO:teuthology.orchestra.run.smithi198.stdout: openblas-threads-0.3.12-1.el8.x86_64 2021-11-11T20:51:04.319 INFO:teuthology.orchestra.run.smithi198.stdout: python3-numpy-1:1.14.3-9.el8.x86_64 2021-11-11T20:51:04.319 INFO:teuthology.orchestra.run.smithi198.stdout: python3-numpy-f2py-1:1.14.3-9.el8.x86_64 2021-11-11T20:51:04.319 INFO:teuthology.orchestra.run.smithi198.stdout: python3-scipy-1.0.0-20.module+el8.1.0+3334+5cb623d7.x86_64 2021-11-11T20:51:04.319 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:51:04.319 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:51:04.453 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph-mgr-rook 2021-11-11T20:51:04.786 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:51:05.305 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 82 kB/s | 2.4 kB 00:00 2021-11-11T20:51:10.176 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:50:25 PM UTC. 2021-11-11T20:51:10.669 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 96 kB/s | 2.8 kB 00:00 2021-11-11T20:51:13.003 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:51:13.003 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:51:13.003 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-diskprediction-local-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:51:13.004 INFO:teuthology.orchestra.run.smithi096.stdout: libgfortran-8.4.1-1.el8.x86_64 2021-11-11T20:51:13.004 INFO:teuthology.orchestra.run.smithi096.stdout: libquadmath-8.4.1-1.el8.x86_64 2021-11-11T20:51:13.005 INFO:teuthology.orchestra.run.smithi096.stdout: openblas-0.3.12-1.el8.x86_64 2021-11-11T20:51:13.005 INFO:teuthology.orchestra.run.smithi096.stdout: openblas-threads-0.3.12-1.el8.x86_64 2021-11-11T20:51:13.005 INFO:teuthology.orchestra.run.smithi096.stdout: python3-numpy-1:1.14.3-9.el8.x86_64 2021-11-11T20:51:13.005 INFO:teuthology.orchestra.run.smithi096.stdout: python3-numpy-f2py-1:1.14.3-9.el8.x86_64 2021-11-11T20:51:13.005 INFO:teuthology.orchestra.run.smithi096.stdout: python3-scipy-1.0.0-20.module+el8.1.0+3334+5cb623d7.x86_64 2021-11-11T20:51:13.006 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:51:13.006 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:51:13.145 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-mgr-rook 2021-11-11T20:51:13.480 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:51:13.996 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 95 kB/s | 2.8 kB 00:00 2021-11-11T20:51:15.775 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:51:18.957 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 79 kB/s | 2.4 kB 00:00 2021-11-11T20:51:19.007 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:51:19.008 INFO:teuthology.orchestra.run.smithi198.stdout:======================================================================================================== 2021-11-11T20:51:19.009 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repository Size 2021-11-11T20:51:19.009 INFO:teuthology.orchestra.run.smithi198.stdout:======================================================================================================== 2021-11-11T20:51:19.009 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:51:19.010 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr-rook noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 46 k 2021-11-11T20:51:19.010 INFO:teuthology.orchestra.run.smithi198.stdout:Installing dependencies: 2021-11-11T20:51:19.010 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2021-11-11T20:51:19.010 INFO:teuthology.orchestra.run.smithi198.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2021-11-11T20:51:19.010 INFO:teuthology.orchestra.run.smithi198.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2021-11-11T20:51:19.011 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2021-11-11T20:51:19.011 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2021-11-11T20:51:19.011 INFO:teuthology.orchestra.run.smithi198.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2021-11-11T20:51:19.011 INFO:teuthology.orchestra.run.smithi198.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2021-11-11T20:51:19.012 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2021-11-11T20:51:19.012 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2021-11-11T20:51:19.012 INFO:teuthology.orchestra.run.smithi198.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2021-11-11T20:51:19.012 INFO:teuthology.orchestra.run.smithi198.stdout: python3-rsa noarch 4.7.2-1.el8 epel 61 k 2021-11-11T20:51:19.013 INFO:teuthology.orchestra.run.smithi198.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2021-11-11T20:51:19.014 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:51:19.014 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:51:19.014 INFO:teuthology.orchestra.run.smithi198.stdout:======================================================================================================== 2021-11-11T20:51:19.015 INFO:teuthology.orchestra.run.smithi198.stdout:Install 13 Packages 2021-11-11T20:51:19.015 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:51:19.018 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 1.9 M 2021-11-11T20:51:19.018 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 25 M 2021-11-11T20:51:19.018 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:51:19.088 INFO:teuthology.orchestra.run.smithi198.stdout:(1/13): python3-cachetools-3.1.1-4.el8.noarch.r 432 kB/s | 33 kB 00:00 2021-11-11T20:51:19.098 INFO:teuthology.orchestra.run.smithi198.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 190 kB/s | 16 kB 00:00 2021-11-11T20:51:19.138 INFO:teuthology.orchestra.run.smithi198.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 1.6 MB/s | 81 kB 00:00 2021-11-11T20:51:19.164 INFO:teuthology.orchestra.run.smithi198.stdout:(4/13): python3-rsa-4.7.2-1.el8.noarch.rpm 2.4 MB/s | 61 kB 00:00 2021-11-11T20:51:19.189 INFO:teuthology.orchestra.run.smithi198.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 2.4 MB/s | 61 kB 00:00 2021-11-11T20:51:19.220 INFO:teuthology.orchestra.run.smithi198.stdout:(6/13): python3-requests-oauthlib-1.0.0-1.el8.n 1.3 MB/s | 43 kB 00:00 2021-11-11T20:51:19.242 INFO:teuthology.orchestra.run.smithi198.stdout:(7/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 7.0 MB/s | 155 kB 00:00 2021-11-11T20:51:19.244 INFO:teuthology.orchestra.run.smithi198.stdout:(8/13): ceph-mgr-rook-17.0.0-8977.ga3a8018b.el8 199 kB/s | 46 kB 00:00 2021-11-11T20:51:19.267 INFO:teuthology.orchestra.run.smithi198.stdout:(9/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 1.1 MB/s | 27 kB 00:00 2021-11-11T20:51:19.280 INFO:teuthology.orchestra.run.smithi198.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 3.0 MB/s | 110 kB 00:00 2021-11-11T20:51:19.297 INFO:teuthology.orchestra.run.smithi198.stdout:(11/13): python3-jsonpointer-1.10-11.el8.noarch 684 kB/s | 20 kB 00:00 2021-11-11T20:51:19.303 INFO:teuthology.orchestra.run.smithi198.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 5.3 MB/s | 126 kB 00:00 2021-11-11T20:51:19.343 INFO:teuthology.orchestra.run.smithi198.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 4.8 MB/s | 1.2 MB 00:00 2021-11-11T20:51:19.343 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:51:19.344 INFO:teuthology.orchestra.run.smithi198.stdout:Total 5.8 MB/s | 1.9 MB 00:00 2021-11-11T20:51:19.354 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:51:19.368 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:51:19.369 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:51:19.436 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:51:19.440 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:51:19.949 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:51:20.207 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2021-11-11T20:51:20.426 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-rsa-4.7.2-1.el8.noarch 2/13 2021-11-11T20:51:20.634 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2021-11-11T20:51:20.861 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2021-11-11T20:51:21.134 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2021-11-11T20:51:21.386 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2021-11-11T20:51:21.677 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2021-11-11T20:51:21.920 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2021-11-11T20:51:22.138 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2021-11-11T20:51:22.350 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2021-11-11T20:51:23.488 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2021-11-11T20:51:23.872 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2021-11-11T20:51:24.043 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-mgr-rook-2:17.0.0-8977.ga3a8018b.el8.noarch 13/13 2021-11-11T20:51:24.276 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-mgr-rook-2:17.0.0-8977.ga3a8018b.el8.noarch 13/13 2021-11-11T20:51:24.277 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-mgr-rook-2:17.0.0-8977.ga3a8018b.el8.noarch 1/13 2021-11-11T20:51:24.277 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2021-11-11T20:51:24.277 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2021-11-11T20:51:24.279 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2021-11-11T20:51:24.279 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2021-11-11T20:51:24.279 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-rsa-4.7.2-1.el8.noarch 6/13 2021-11-11T20:51:24.280 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2021-11-11T20:51:24.280 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2021-11-11T20:51:24.280 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2021-11-11T20:51:24.280 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 10/13 2021-11-11T20:51:24.281 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 11/13 2021-11-11T20:51:24.281 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 12/13 2021-11-11T20:51:24.380 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:51:24.603 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 13/13 2021-11-11T20:51:24.603 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:51:27.094 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:51:27.095 INFO:teuthology.orchestra.run.smithi096.stdout:======================================================================================================== 2021-11-11T20:51:27.096 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2021-11-11T20:51:27.096 INFO:teuthology.orchestra.run.smithi096.stdout:======================================================================================================== 2021-11-11T20:51:27.096 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:51:27.096 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-rook noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 46 k 2021-11-11T20:51:27.097 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2021-11-11T20:51:27.097 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2021-11-11T20:51:27.097 INFO:teuthology.orchestra.run.smithi096.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2021-11-11T20:51:27.097 INFO:teuthology.orchestra.run.smithi096.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2021-11-11T20:51:27.098 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2021-11-11T20:51:27.098 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2021-11-11T20:51:27.098 INFO:teuthology.orchestra.run.smithi096.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2021-11-11T20:51:27.098 INFO:teuthology.orchestra.run.smithi096.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2021-11-11T20:51:27.099 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2021-11-11T20:51:27.099 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2021-11-11T20:51:27.099 INFO:teuthology.orchestra.run.smithi096.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2021-11-11T20:51:27.099 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rsa noarch 4.7.2-1.el8 epel 61 k 2021-11-11T20:51:27.099 INFO:teuthology.orchestra.run.smithi096.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2021-11-11T20:51:27.100 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:51:27.100 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:51:27.101 INFO:teuthology.orchestra.run.smithi096.stdout:======================================================================================================== 2021-11-11T20:51:27.101 INFO:teuthology.orchestra.run.smithi096.stdout:Install 13 Packages 2021-11-11T20:51:27.101 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:51:27.102 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 1.9 M 2021-11-11T20:51:27.102 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 25 M 2021-11-11T20:51:27.102 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:51:27.173 INFO:teuthology.orchestra.run.smithi096.stdout:(1/13): python3-cachetools-3.1.1-4.el8.noarch.r 446 kB/s | 33 kB 00:00 2021-11-11T20:51:27.173 INFO:teuthology.orchestra.run.smithi096.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 220 kB/s | 16 kB 00:00 2021-11-11T20:51:27.223 INFO:teuthology.orchestra.run.smithi096.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 1.6 MB/s | 81 kB 00:00 2021-11-11T20:51:27.248 INFO:teuthology.orchestra.run.smithi096.stdout:(4/13): python3-rsa-4.7.2-1.el8.noarch.rpm 2.4 MB/s | 61 kB 00:00 2021-11-11T20:51:27.274 INFO:teuthology.orchestra.run.smithi096.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 2.4 MB/s | 61 kB 00:00 2021-11-11T20:51:27.305 INFO:teuthology.orchestra.run.smithi096.stdout:(6/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 852 kB/s | 27 kB 00:00 2021-11-11T20:51:27.312 INFO:teuthology.orchestra.run.smithi096.stdout:(7/13): ceph-mgr-rook-17.0.0-8977.ga3a8018b.el8 216 kB/s | 46 kB 00:00 2021-11-11T20:51:27.325 INFO:teuthology.orchestra.run.smithi096.stdout:(8/13): python3-pyasn1-modules-0.3.7-6.el8.noar 5.4 MB/s | 110 kB 00:00 2021-11-11T20:51:27.349 INFO:teuthology.orchestra.run.smithi096.stdout:(9/13): python3-kubernetes-11.0.0-6.el8.noarch. 6.6 MB/s | 1.2 MB 00:00 2021-11-11T20:51:27.350 INFO:teuthology.orchestra.run.smithi096.stdout:(10/13): python3-jsonpointer-1.10-11.el8.noarch 526 kB/s | 20 kB 00:00 2021-11-11T20:51:27.352 INFO:teuthology.orchestra.run.smithi096.stdout:(11/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 4.5 MB/s | 126 kB 00:00 2021-11-11T20:51:27.381 INFO:teuthology.orchestra.run.smithi096.stdout:(12/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 5.1 MB/s | 155 kB 00:00 2021-11-11T20:51:27.386 INFO:teuthology.orchestra.run.smithi096.stdout:(13/13): python3-requests-oauthlib-1.0.0-1.el8. 1.1 MB/s | 43 kB 00:00 2021-11-11T20:51:27.387 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:51:27.387 INFO:teuthology.orchestra.run.smithi096.stdout:Total 6.6 MB/s | 1.9 MB 00:00 2021-11-11T20:51:27.397 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:51:27.411 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:51:27.412 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:51:27.479 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:51:27.482 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:51:27.989 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:51:28.240 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2021-11-11T20:51:28.476 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-rsa-4.7.2-1.el8.noarch 2/13 2021-11-11T20:51:28.780 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2021-11-11T20:51:29.051 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2021-11-11T20:51:29.227 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2021-11-11T20:51:29.445 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2021-11-11T20:51:29.668 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2021-11-11T20:51:29.845 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2021-11-11T20:51:30.097 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2021-11-11T20:51:30.361 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2021-11-11T20:51:31.522 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2021-11-11T20:51:32.080 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2021-11-11T20:51:32.261 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-rook-2:17.0.0-8977.ga3a8018b.el8.noarch 13/13 2021-11-11T20:51:32.483 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mgr-rook-2:17.0.0-8977.ga3a8018b.el8.noarch 13/13 2021-11-11T20:51:32.484 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-rook-2:17.0.0-8977.ga3a8018b.el8.noarch 1/13 2021-11-11T20:51:32.484 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2021-11-11T20:51:32.484 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2021-11-11T20:51:32.484 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2021-11-11T20:51:32.485 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2021-11-11T20:51:32.485 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-rsa-4.7.2-1.el8.noarch 6/13 2021-11-11T20:51:32.485 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2021-11-11T20:51:32.485 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 8/13 2021-11-11T20:51:32.485 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 9/13 2021-11-11T20:51:32.486 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 10/13 2021-11-11T20:51:32.486 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 11/13 2021-11-11T20:51:32.486 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2021-11-11T20:51:32.811 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2021-11-11T20:51:32.812 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:51:36.495 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:50:55 PM UTC. 2021-11-11T20:51:39.267 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:51:39.267 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:51:39.267 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr-rook-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:51:39.268 INFO:teuthology.orchestra.run.smithi198.stdout: python3-cachetools-3.1.1-4.el8.noarch 2021-11-11T20:51:39.268 INFO:teuthology.orchestra.run.smithi198.stdout: python3-certifi-2018.10.15-7.el8.noarch 2021-11-11T20:51:39.268 INFO:teuthology.orchestra.run.smithi198.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2021-11-11T20:51:39.268 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jsonpatch-1.21-2.el8.noarch 2021-11-11T20:51:39.269 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jsonpointer-1.10-11.el8.noarch 2021-11-11T20:51:39.269 INFO:teuthology.orchestra.run.smithi198.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2021-11-11T20:51:39.269 INFO:teuthology.orchestra.run.smithi198.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2021-11-11T20:51:39.269 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2021-11-11T20:51:39.270 INFO:teuthology.orchestra.run.smithi198.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2021-11-11T20:51:39.270 INFO:teuthology.orchestra.run.smithi198.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2021-11-11T20:51:39.270 INFO:teuthology.orchestra.run.smithi198.stdout: python3-rsa-4.7.2-1.el8.noarch 2021-11-11T20:51:39.270 INFO:teuthology.orchestra.run.smithi198.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2021-11-11T20:51:39.270 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:51:39.271 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:51:39.390 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph-mgr-cephadm 2021-11-11T20:51:39.727 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:51:40.258 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 81 kB/s | 2.4 kB 00:00 2021-11-11T20:51:44.535 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:51:04 PM UTC. 2021-11-11T20:51:45.571 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 97 kB/s | 2.8 kB 00:00 2021-11-11T20:51:47.355 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:51:47.355 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:51:47.355 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-rook-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:51:47.356 INFO:teuthology.orchestra.run.smithi096.stdout: python3-cachetools-3.1.1-4.el8.noarch 2021-11-11T20:51:47.356 INFO:teuthology.orchestra.run.smithi096.stdout: python3-certifi-2018.10.15-7.el8.noarch 2021-11-11T20:51:47.356 INFO:teuthology.orchestra.run.smithi096.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2021-11-11T20:51:47.356 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jsonpatch-1.21-2.el8.noarch 2021-11-11T20:51:47.357 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jsonpointer-1.10-11.el8.noarch 2021-11-11T20:51:47.357 INFO:teuthology.orchestra.run.smithi096.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2021-11-11T20:51:47.357 INFO:teuthology.orchestra.run.smithi096.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2021-11-11T20:51:47.357 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2021-11-11T20:51:47.358 INFO:teuthology.orchestra.run.smithi096.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2021-11-11T20:51:47.358 INFO:teuthology.orchestra.run.smithi096.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2021-11-11T20:51:47.358 INFO:teuthology.orchestra.run.smithi096.stdout: python3-rsa-4.7.2-1.el8.noarch 2021-11-11T20:51:47.358 INFO:teuthology.orchestra.run.smithi096.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2021-11-11T20:51:47.358 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:51:47.359 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:51:47.502 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-mgr-cephadm 2021-11-11T20:51:47.839 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:51:48.343 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 95 kB/s | 2.8 kB 00:00 2021-11-11T20:51:50.646 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:51:53.268 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:51:53.269 INFO:teuthology.orchestra.run.smithi198.stdout:=============================================================================================== 2021-11-11T20:51:53.269 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repository Size 2021-11-11T20:51:53.269 INFO:teuthology.orchestra.run.smithi198.stdout:=============================================================================================== 2021-11-11T20:51:53.270 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:51:53.270 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr-cephadm noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 99 k 2021-11-11T20:51:53.270 INFO:teuthology.orchestra.run.smithi198.stdout:Installing dependencies: 2021-11-11T20:51:53.270 INFO:teuthology.orchestra.run.smithi198.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2021-11-11T20:51:53.271 INFO:teuthology.orchestra.run.smithi198.stdout: python3-babel noarch 2.5.1-5.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2021-11-11T20:51:53.271 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jinja2 noarch 2.10.1-2.el8_0 rhel-8-for-x86_64-appstream-rpms 538 k 2021-11-11T20:51:53.271 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:51:53.272 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:51:53.272 INFO:teuthology.orchestra.run.smithi198.stdout:=============================================================================================== 2021-11-11T20:51:53.272 INFO:teuthology.orchestra.run.smithi198.stdout:Install 4 Packages 2021-11-11T20:51:53.273 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:51:53.273 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 5.9 M 2021-11-11T20:51:53.273 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 25 M 2021-11-11T20:51:53.273 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:51:53.303 INFO:teuthology.orchestra.run.smithi198.stdout:(1/4): python3-jinja2-2.10.1-2.el8_0.noarch.rpm 16 MB/s | 538 kB 00:00 2021-11-11T20:51:53.355 INFO:teuthology.orchestra.run.smithi198.stdout:(2/4): python3-babel-2.5.1-5.el8.noarch.rpm 92 MB/s | 4.8 MB 00:00 2021-11-11T20:51:53.361 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 80 kB/s | 2.4 kB 00:00 2021-11-11T20:51:53.441 INFO:teuthology.orchestra.run.smithi198.stdout:(3/4): python3-asyncssh-2.7.0-2.el8.noarch.rpm 2.7 MB/s | 476 kB 00:00 2021-11-11T20:51:53.525 INFO:teuthology.orchestra.run.smithi198.stdout:(4/4): ceph-mgr-cephadm-17.0.0-8977.ga3a8018b.e 391 kB/s | 99 kB 00:00 2021-11-11T20:51:53.525 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:51:53.526 INFO:teuthology.orchestra.run.smithi198.stdout:Total 23 MB/s | 5.9 MB 00:00 2021-11-11T20:51:53.531 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:51:53.539 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:51:53.540 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:51:53.614 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:51:53.618 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:51:54.619 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:51:55.070 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-babel-2.5.1-5.el8.noarch 1/4 2021-11-11T20:51:55.439 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-jinja2-2.10.1-2.el8_0.noarch 2/4 2021-11-11T20:51:55.730 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 3/4 2021-11-11T20:51:55.888 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-mgr-cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 4/4 2021-11-11T20:51:56.130 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 4/4 2021-11-11T20:51:56.133 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-mgr-cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 1/4 2021-11-11T20:51:56.133 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/4 2021-11-11T20:51:56.133 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-jinja2-2.10.1-2.el8_0.noarch 3/4 2021-11-11T20:51:56.362 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : python3-babel-2.5.1-5.el8.noarch 4/4 2021-11-11T20:51:56.362 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:51:58.763 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:52:01.382 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:52:01.383 INFO:teuthology.orchestra.run.smithi096.stdout:=============================================================================================== 2021-11-11T20:52:01.383 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2021-11-11T20:52:01.384 INFO:teuthology.orchestra.run.smithi096.stdout:=============================================================================================== 2021-11-11T20:52:01.384 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:52:01.384 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-cephadm noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 99 k 2021-11-11T20:52:01.384 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2021-11-11T20:52:01.385 INFO:teuthology.orchestra.run.smithi096.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2021-11-11T20:52:01.385 INFO:teuthology.orchestra.run.smithi096.stdout: python3-babel noarch 2.5.1-5.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2021-11-11T20:52:01.385 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jinja2 noarch 2.10.1-2.el8_0 rhel-8-for-x86_64-appstream-rpms 538 k 2021-11-11T20:52:01.385 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:52:01.386 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:52:01.386 INFO:teuthology.orchestra.run.smithi096.stdout:=============================================================================================== 2021-11-11T20:52:01.386 INFO:teuthology.orchestra.run.smithi096.stdout:Install 4 Packages 2021-11-11T20:52:01.386 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:52:01.387 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 5.9 M 2021-11-11T20:52:01.387 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 25 M 2021-11-11T20:52:01.388 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:52:01.419 INFO:teuthology.orchestra.run.smithi096.stdout:(1/4): python3-jinja2-2.10.1-2.el8_0.noarch.rpm 15 MB/s | 538 kB 00:00 2021-11-11T20:52:01.473 INFO:teuthology.orchestra.run.smithi096.stdout:(2/4): python3-babel-2.5.1-5.el8.noarch.rpm 90 MB/s | 4.8 MB 00:00 2021-11-11T20:52:01.579 INFO:teuthology.orchestra.run.smithi096.stdout:(3/4): python3-asyncssh-2.7.0-2.el8.noarch.rpm 2.4 MB/s | 476 kB 00:00 2021-11-11T20:52:01.629 INFO:teuthology.orchestra.run.smithi096.stdout:(4/4): ceph-mgr-cephadm-17.0.0-8977.ga3a8018b.e 409 kB/s | 99 kB 00:00 2021-11-11T20:52:01.630 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:52:01.630 INFO:teuthology.orchestra.run.smithi096.stdout:Total 24 MB/s | 5.9 MB 00:00 2021-11-11T20:52:01.635 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:52:01.645 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:52:01.645 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:52:01.720 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:52:01.724 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:52:02.673 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:52:03.103 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-babel-2.5.1-5.el8.noarch 1/4 2021-11-11T20:52:03.461 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-jinja2-2.10.1-2.el8_0.noarch 2/4 2021-11-11T20:52:03.756 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 3/4 2021-11-11T20:52:03.956 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-mgr-cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 4/4 2021-11-11T20:52:04.168 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 4/4 2021-11-11T20:52:04.168 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-mgr-cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 1/4 2021-11-11T20:52:04.169 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/4 2021-11-11T20:52:04.169 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-jinja2-2.10.1-2.el8_0.noarch 3/4 2021-11-11T20:52:04.440 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : python3-babel-2.5.1-5.el8.noarch 4/4 2021-11-11T20:52:04.441 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:52:08.248 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:51:30 PM UTC. 2021-11-11T20:52:11.006 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:52:11.006 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:52:11.007 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-mgr-cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:52:11.007 INFO:teuthology.orchestra.run.smithi198.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2021-11-11T20:52:11.008 INFO:teuthology.orchestra.run.smithi198.stdout: python3-babel-2.5.1-5.el8.noarch 2021-11-11T20:52:11.008 INFO:teuthology.orchestra.run.smithi198.stdout: python3-jinja2-2.10.1-2.el8_0.noarch 2021-11-11T20:52:11.008 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:52:11.008 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:52:11.170 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph-fuse 2021-11-11T20:52:11.510 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:52:12.061 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 78 kB/s | 2.4 kB 00:00 2021-11-11T20:52:16.161 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:51:38 PM UTC. 2021-11-11T20:52:17.462 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 91 kB/s | 2.8 kB 00:00 2021-11-11T20:52:18.882 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:52:18.882 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:52:18.882 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-mgr-cephadm-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:52:18.883 INFO:teuthology.orchestra.run.smithi096.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2021-11-11T20:52:18.883 INFO:teuthology.orchestra.run.smithi096.stdout: python3-babel-2.5.1-5.el8.noarch 2021-11-11T20:52:18.883 INFO:teuthology.orchestra.run.smithi096.stdout: python3-jinja2-2.10.1-2.el8_0.noarch 2021-11-11T20:52:18.883 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:52:18.883 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:52:18.998 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-fuse 2021-11-11T20:52:19.332 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:52:19.833 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 97 kB/s | 2.8 kB 00:00 2021-11-11T20:52:22.520 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:52:24.807 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 80 kB/s | 2.4 kB 00:00 2021-11-11T20:52:25.077 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:52:25.078 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:52:25.078 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repository Size 2021-11-11T20:52:25.078 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:52:25.079 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:52:25.079 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-fuse 2021-11-11T20:52:25.079 INFO:teuthology.orchestra.run.smithi198.stdout: x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 871 k 2021-11-11T20:52:25.079 INFO:teuthology.orchestra.run.smithi198.stdout:Installing dependencies: 2021-11-11T20:52:25.080 INFO:teuthology.orchestra.run.smithi198.stdout: fuse x86_64 2.9.7-12.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2021-11-11T20:52:25.080 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:52:25.080 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:52:25.080 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:52:25.081 INFO:teuthology.orchestra.run.smithi198.stdout:Install 2 Packages 2021-11-11T20:52:25.081 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:52:25.081 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 954 k 2021-11-11T20:52:25.081 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 3.1 M 2021-11-11T20:52:25.082 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:52:25.109 INFO:teuthology.orchestra.run.smithi198.stdout:(1/2): fuse-2.9.7-12.el8.x86_64.rpm 2.7 MB/s | 83 kB 00:00 2021-11-11T20:52:25.525 INFO:teuthology.orchestra.run.smithi198.stdout:(2/2): ceph-fuse-17.0.0-8977.ga3a8018b.el8.x86_ 1.9 MB/s | 871 kB 00:00 2021-11-11T20:52:25.526 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:52:25.526 INFO:teuthology.orchestra.run.smithi198.stdout:Total 2.1 MB/s | 954 kB 00:00 2021-11-11T20:52:25.528 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:52:25.550 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:52:25.550 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:52:25.622 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:52:25.626 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:52:26.051 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:52:26.334 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : fuse-2.9.7-12.el8.x86_64 1/2 2021-11-11T20:52:26.532 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/2 2021-11-11T20:52:26.869 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/2 2021-11-11T20:52:26.869 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/2 2021-11-11T20:52:27.122 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : fuse-2.9.7-12.el8.x86_64 2/2 2021-11-11T20:52:27.122 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:52:30.180 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:52:32.815 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:52:32.816 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:52:32.816 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2021-11-11T20:52:32.816 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:52:32.817 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:52:32.817 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-fuse 2021-11-11T20:52:32.817 INFO:teuthology.orchestra.run.smithi096.stdout: x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 871 k 2021-11-11T20:52:32.817 INFO:teuthology.orchestra.run.smithi096.stdout:Installing dependencies: 2021-11-11T20:52:32.817 INFO:teuthology.orchestra.run.smithi096.stdout: fuse x86_64 2.9.7-12.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2021-11-11T20:52:32.818 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:52:32.818 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:52:32.819 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:52:32.819 INFO:teuthology.orchestra.run.smithi096.stdout:Install 2 Packages 2021-11-11T20:52:32.819 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:52:32.820 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 954 k 2021-11-11T20:52:32.820 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 3.1 M 2021-11-11T20:52:32.820 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:52:32.847 INFO:teuthology.orchestra.run.smithi096.stdout:(1/2): fuse-2.9.7-12.el8.x86_64.rpm 2.7 MB/s | 83 kB 00:00 2021-11-11T20:52:33.329 INFO:teuthology.orchestra.run.smithi096.stdout:(2/2): ceph-fuse-17.0.0-8977.ga3a8018b.el8.x86_ 1.7 MB/s | 871 kB 00:00 2021-11-11T20:52:33.330 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:52:33.331 INFO:teuthology.orchestra.run.smithi096.stdout:Total 1.8 MB/s | 954 kB 00:00 2021-11-11T20:52:33.332 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:52:33.354 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:52:33.355 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:52:33.427 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:52:33.431 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:52:33.588 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:52:34.227 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : fuse-2.9.7-12.el8.x86_64 1/2 2021-11-11T20:52:34.413 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/2 2021-11-11T20:52:34.733 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 2/2 2021-11-11T20:52:34.733 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/2 2021-11-11T20:52:35.005 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : fuse-2.9.7-12.el8.x86_64 2/2 2021-11-11T20:52:35.006 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:52:39.018 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:52:02 PM UTC. 2021-11-11T20:52:41.865 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:52:41.865 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:52:41.865 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 fuse-2.9.7-12.el8.x86_64 2021-11-11T20:52:41.865 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:52:41.866 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:52:41.996 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install ceph-volume 2021-11-11T20:52:42.336 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:52:42.865 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 82 kB/s | 2.4 kB 00:00 2021-11-11T20:52:46.751 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:52:10 PM UTC. 2021-11-11T20:52:48.165 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 91 kB/s | 2.8 kB 00:00 2021-11-11T20:52:49.547 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:52:49.549 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:52:49.549 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 fuse-2.9.7-12.el8.x86_64 2021-11-11T20:52:49.549 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:52:49.550 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:52:49.695 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install ceph-volume 2021-11-11T20:52:50.034 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:52:50.544 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 93 kB/s | 2.8 kB 00:00 2021-11-11T20:52:53.145 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:52:55.549 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 82 kB/s | 2.4 kB 00:00 2021-11-11T20:52:55.766 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:52:55.767 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:52:55.767 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repository Size 2021-11-11T20:52:55.767 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:52:55.768 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:52:55.768 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-volume noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 255 k 2021-11-11T20:52:55.768 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:52:55.768 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:52:55.769 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:52:55.769 INFO:teuthology.orchestra.run.smithi198.stdout:Install 1 Package 2021-11-11T20:52:55.769 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:52:55.769 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 255 k 2021-11-11T20:52:55.769 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 1.2 M 2021-11-11T20:52:55.770 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:52:56.143 INFO:teuthology.orchestra.run.smithi198.stdout:ceph-volume-17.0.0-8977.ga3a8018b.el8.noarch.rp 680 kB/s | 255 kB 00:00 2021-11-11T20:52:56.145 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:52:56.145 INFO:teuthology.orchestra.run.smithi198.stdout:Total 677 kB/s | 255 kB 00:00 2021-11-11T20:52:56.146 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:52:56.155 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:52:56.156 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:52:56.172 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:52:56.176 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:52:56.356 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:52:56.629 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : ceph-volume-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:52:56.970 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: ceph-volume-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:52:57.185 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : ceph-volume-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:52:57.185 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:53:00.989 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:53:03.608 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:53:03.609 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:53:03.610 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repository Size 2021-11-11T20:53:03.610 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:53:03.610 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:53:03.610 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-volume noarch 2:17.0.0-8977.ga3a8018b.el8 ceph-noarch 255 k 2021-11-11T20:53:03.611 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:53:03.611 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:53:03.611 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:53:03.611 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2021-11-11T20:53:03.612 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:53:03.612 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 255 k 2021-11-11T20:53:03.612 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 1.2 M 2021-11-11T20:53:03.612 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:53:03.891 INFO:teuthology.orchestra.run.smithi096.stdout:ceph-volume-17.0.0-8977.ga3a8018b.el8.noarch.rp 911 kB/s | 255 kB 00:00 2021-11-11T20:53:03.892 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:53:03.892 INFO:teuthology.orchestra.run.smithi096.stdout:Total 906 kB/s | 255 kB 00:00 2021-11-11T20:53:03.892 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:53:03.902 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:53:03.903 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:53:03.919 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:53:03.923 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:53:04.130 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:53:04.410 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : ceph-volume-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:53:04.748 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: ceph-volume-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:53:04.982 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : ceph-volume-2:17.0.0-8977.ga3a8018b.el8.noarch 1/1 2021-11-11T20:53:04.982 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:53:09.298 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:52:33 PM UTC. 2021-11-11T20:53:12.067 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:53:12.067 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:53:12.068 INFO:teuthology.orchestra.run.smithi198.stdout: ceph-volume-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:53:12.068 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:53:12.068 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:53:12.193 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install librados-devel 2021-11-11T20:53:12.534 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:53:13.225 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 80 kB/s | 2.4 kB 00:00 2021-11-11T20:53:16.768 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:52:41 PM UTC. 2021-11-11T20:53:18.560 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 91 kB/s | 2.8 kB 00:00 2021-11-11T20:53:19.483 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:53:19.484 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:53:19.484 INFO:teuthology.orchestra.run.smithi096.stdout: ceph-volume-2:17.0.0-8977.ga3a8018b.el8.noarch 2021-11-11T20:53:19.484 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:53:19.484 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:53:19.602 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install librados-devel 2021-11-11T20:53:19.939 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:53:20.474 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 81 kB/s | 2.8 kB 00:00 2021-11-11T20:53:23.512 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:53:25.478 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 81 kB/s | 2.4 kB 00:00 2021-11-11T20:53:26.175 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:53:26.176 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:53:26.176 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repo Size 2021-11-11T20:53:26.177 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:53:26.177 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:53:26.177 INFO:teuthology.orchestra.run.smithi198.stdout: librados-devel x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 125 k 2021-11-11T20:53:26.177 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:53:26.177 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:53:26.178 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:53:26.178 INFO:teuthology.orchestra.run.smithi198.stdout:Install 1 Package 2021-11-11T20:53:26.178 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:53:26.178 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 125 k 2021-11-11T20:53:26.179 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 432 k 2021-11-11T20:53:26.179 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:53:26.419 INFO:teuthology.orchestra.run.smithi198.stdout:librados-devel-17.0.0-8977.ga3a8018b.el8.x86_64 516 kB/s | 125 kB 00:00 2021-11-11T20:53:26.420 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:53:26.421 INFO:teuthology.orchestra.run.smithi198.stdout:Total 513 kB/s | 125 kB 00:00 2021-11-11T20:53:26.421 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:53:26.427 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:53:26.427 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:53:26.481 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:53:26.485 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:53:26.619 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:53:26.779 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : librados-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:53:27.012 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: librados-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:53:27.287 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : librados-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:53:27.288 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:53:31.122 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:53:33.713 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:53:33.714 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:53:33.715 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repo Size 2021-11-11T20:53:33.715 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:53:33.715 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:53:33.715 INFO:teuthology.orchestra.run.smithi096.stdout: librados-devel x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 125 k 2021-11-11T20:53:33.716 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:53:33.716 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:53:33.716 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:53:33.716 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2021-11-11T20:53:33.716 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:53:33.717 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 125 k 2021-11-11T20:53:33.717 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 432 k 2021-11-11T20:53:33.717 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:53:33.982 INFO:teuthology.orchestra.run.smithi096.stdout:librados-devel-17.0.0-8977.ga3a8018b.el8.x86_64 468 kB/s | 125 kB 00:00 2021-11-11T20:53:33.984 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:53:33.984 INFO:teuthology.orchestra.run.smithi096.stdout:Total 465 kB/s | 125 kB 00:00 2021-11-11T20:53:33.986 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:53:33.991 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:53:33.991 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:53:34.046 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:53:34.050 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:53:34.162 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:53:34.348 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : librados-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:53:34.583 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: librados-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:53:34.829 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : librados-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:53:34.829 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:53:39.157 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:53:03 PM UTC. 2021-11-11T20:53:41.966 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:53:41.967 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:53:41.967 INFO:teuthology.orchestra.run.smithi198.stdout: librados-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:53:41.968 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:53:41.968 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:53:42.090 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install libcephfs2 2021-11-11T20:53:42.429 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:53:42.953 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 78 kB/s | 2.4 kB 00:00 2021-11-11T20:53:46.933 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:53:11 PM UTC. 2021-11-11T20:53:48.314 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 95 kB/s | 2.8 kB 00:00 2021-11-11T20:53:49.659 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:53:49.659 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:53:49.660 INFO:teuthology.orchestra.run.smithi096.stdout: librados-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:53:49.660 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:53:49.660 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:53:49.781 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install libcephfs2 2021-11-11T20:53:50.116 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:53:50.624 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 92 kB/s | 2.8 kB 00:00 2021-11-11T20:53:53.406 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:53:55.640 INFO:teuthology.orchestra.run.smithi198.stdout:Package libcephfs2-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:53:55.665 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 81 kB/s | 2.4 kB 00:00 2021-11-11T20:53:56.081 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:53:56.082 INFO:teuthology.orchestra.run.smithi198.stdout:Nothing to do. 2021-11-11T20:53:56.083 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:53:56.192 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install libcephfs-devel 2021-11-11T20:53:56.528 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:53:57.070 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 77 kB/s | 2.4 kB 00:00 2021-11-11T20:53:57.202 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 88 kB/s | 2.8 kB 00:00 2021-11-11T20:53:59.117 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:53:59.118 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:53:59.118 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repo Size 2021-11-11T20:53:59.118 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:53:59.119 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:53:59.119 INFO:teuthology.orchestra.run.smithi198.stdout: libcephfs-devel x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 24 k 2021-11-11T20:53:59.119 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:53:59.119 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:53:59.120 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:53:59.120 INFO:teuthology.orchestra.run.smithi198.stdout:Install 1 Package 2021-11-11T20:53:59.120 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:53:59.120 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 24 k 2021-11-11T20:53:59.121 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 105 k 2021-11-11T20:53:59.121 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:53:59.312 INFO:teuthology.orchestra.run.smithi198.stdout:libcephfs-devel-17.0.0-8977.ga3a8018b.el8.x86_6 125 kB/s | 24 kB 00:00 2021-11-11T20:53:59.313 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:53:59.313 INFO:teuthology.orchestra.run.smithi198.stdout:Total 124 kB/s | 24 kB 00:00 2021-11-11T20:53:59.313 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:53:59.318 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:53:59.318 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:53:59.319 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:53:59.324 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:53:59.384 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:53:59.565 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : libcephfs-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:53:59.756 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: libcephfs-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:54:00.002 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : libcephfs-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:54:00.003 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:54:01.115 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:54:03.248 INFO:teuthology.orchestra.run.smithi096.stdout:Package libcephfs2-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:03.687 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:54:03.688 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2021-11-11T20:54:03.689 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:54:03.809 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install libcephfs-devel 2021-11-11T20:54:04.143 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:04.965 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 97 kB/s | 2.8 kB 00:00 2021-11-11T20:54:05.146 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 81 kB/s | 2.4 kB 00:00 2021-11-11T20:54:06.996 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:54:06.997 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:54:06.997 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repo Size 2021-11-11T20:54:06.997 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:54:06.997 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:54:06.998 INFO:teuthology.orchestra.run.smithi096.stdout: libcephfs-devel x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 24 k 2021-11-11T20:54:06.998 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:54:06.998 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:54:06.998 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:54:06.999 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2021-11-11T20:54:06.999 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:54:06.999 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 24 k 2021-11-11T20:54:06.999 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 105 k 2021-11-11T20:54:07.000 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:54:07.173 INFO:teuthology.orchestra.run.smithi096.stdout:libcephfs-devel-17.0.0-8977.ga3a8018b.el8.x86_6 137 kB/s | 24 kB 00:00 2021-11-11T20:54:07.174 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:54:07.175 INFO:teuthology.orchestra.run.smithi096.stdout:Total 136 kB/s | 24 kB 00:00 2021-11-11T20:54:07.175 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:54:07.179 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:54:07.179 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:54:07.181 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:54:07.185 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:54:07.234 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:54:07.426 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : libcephfs-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:54:07.627 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: libcephfs-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:54:07.845 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : libcephfs-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:54:07.846 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:54:11.806 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:53:33 PM UTC. 2021-11-11T20:54:14.543 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:54:14.543 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:54:14.544 INFO:teuthology.orchestra.run.smithi198.stdout: libcephfs-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:54:14.544 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:54:14.545 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:54:14.667 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install librados2 2021-11-11T20:54:15.004 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:15.552 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 83 kB/s | 2.4 kB 00:00 2021-11-11T20:54:19.435 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:53:41 PM UTC. 2021-11-11T20:54:20.828 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 98 kB/s | 2.8 kB 00:00 2021-11-11T20:54:22.180 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:54:22.180 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:54:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: libcephfs-devel-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:54:22.181 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:54:22.181 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:54:22.335 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install librados2 2021-11-11T20:54:22.672 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:23.241 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 73 kB/s | 2.8 kB 00:00 2021-11-11T20:54:25.827 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:54:28.084 INFO:teuthology.orchestra.run.smithi198.stdout:Package librados2-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:28.223 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 78 kB/s | 2.4 kB 00:00 2021-11-11T20:54:28.531 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:54:28.532 INFO:teuthology.orchestra.run.smithi198.stdout:Nothing to do. 2021-11-11T20:54:28.532 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:54:28.643 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install librbd1 2021-11-11T20:54:28.990 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:29.536 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 74 kB/s | 2.4 kB 00:00 2021-11-11T20:54:29.667 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 88 kB/s | 2.8 kB 00:00 2021-11-11T20:54:31.100 INFO:teuthology.orchestra.run.smithi198.stdout:Package librbd1-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:31.545 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:54:31.546 INFO:teuthology.orchestra.run.smithi198.stdout:Nothing to do. 2021-11-11T20:54:31.546 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:54:31.651 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install python3-rados 2021-11-11T20:54:31.984 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:32.505 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 79 kB/s | 2.4 kB 00:00 2021-11-11T20:54:32.630 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 97 kB/s | 2.8 kB 00:00 2021-11-11T20:54:33.827 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:54:34.083 INFO:teuthology.orchestra.run.smithi198.stdout:Package python3-rados-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:34.523 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:54:34.524 INFO:teuthology.orchestra.run.smithi198.stdout:Nothing to do. 2021-11-11T20:54:34.524 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:54:34.646 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install python3-rgw 2021-11-11T20:54:34.981 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:35.566 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 80 kB/s | 2.4 kB 00:00 2021-11-11T20:54:35.695 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 95 kB/s | 2.8 kB 00:00 2021-11-11T20:54:36.033 INFO:teuthology.orchestra.run.smithi096.stdout:Package librados2-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:36.468 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:54:36.469 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2021-11-11T20:54:36.469 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:54:36.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install librbd1 2021-11-11T20:54:36.959 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:37.158 INFO:teuthology.orchestra.run.smithi198.stdout:Package python3-rgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:37.479 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 95 kB/s | 2.8 kB 00:00 2021-11-11T20:54:37.599 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:54:37.602 INFO:teuthology.orchestra.run.smithi198.stdout:Nothing to do. 2021-11-11T20:54:37.602 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:54:37.657 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 80 kB/s | 2.4 kB 00:00 2021-11-11T20:54:37.703 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install python3-cephfs 2021-11-11T20:54:38.036 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:38.563 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 81 kB/s | 2.4 kB 00:00 2021-11-11T20:54:38.688 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 95 kB/s | 2.8 kB 00:00 2021-11-11T20:54:39.049 INFO:teuthology.orchestra.run.smithi096.stdout:Package librbd1-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:39.491 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:54:39.492 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2021-11-11T20:54:39.492 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:54:39.604 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install python3-rados 2021-11-11T20:54:39.940 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:40.149 INFO:teuthology.orchestra.run.smithi198.stdout:Package python3-cephfs-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:40.435 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 95 kB/s | 2.8 kB 00:00 2021-11-11T20:54:40.588 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:54:40.590 INFO:teuthology.orchestra.run.smithi198.stdout:Nothing to do. 2021-11-11T20:54:40.590 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:54:40.612 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 82 kB/s | 2.4 kB 00:00 2021-11-11T20:54:40.693 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install python3-rbd 2021-11-11T20:54:41.028 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:41.566 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 73 kB/s | 2.4 kB 00:00 2021-11-11T20:54:41.696 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 95 kB/s | 2.8 kB 00:00 2021-11-11T20:54:42.006 INFO:teuthology.orchestra.run.smithi096.stdout:Package python3-rados-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:42.451 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:54:42.452 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2021-11-11T20:54:42.452 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:54:42.572 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install python3-rgw 2021-11-11T20:54:42.907 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:43.115 INFO:teuthology.orchestra.run.smithi198.stdout:Package python3-rbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:43.430 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 97 kB/s | 2.8 kB 00:00 2021-11-11T20:54:43.555 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:54:43.556 INFO:teuthology.orchestra.run.smithi198.stdout:Nothing to do. 2021-11-11T20:54:43.557 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:54:43.611 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 76 kB/s | 2.4 kB 00:00 2021-11-11T20:54:43.659 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install rbd-fuse 2021-11-11T20:54:43.995 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:44.550 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 79 kB/s | 2.4 kB 00:00 2021-11-11T20:54:44.677 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 97 kB/s | 2.8 kB 00:00 2021-11-11T20:54:44.997 INFO:teuthology.orchestra.run.smithi096.stdout:Package python3-rgw-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:45.438 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:54:45.439 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2021-11-11T20:54:45.439 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:54:45.553 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install python3-cephfs 2021-11-11T20:54:45.889 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:46.403 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 97 kB/s | 2.8 kB 00:00 2021-11-11T20:54:46.554 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:54:46.555 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:54:46.556 INFO:teuthology.orchestra.run.smithi198.stdout: Package Architecture Version Repository Size 2021-11-11T20:54:46.556 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:54:46.556 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:54:46.556 INFO:teuthology.orchestra.run.smithi198.stdout: rbd-fuse x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 93 k 2021-11-11T20:54:46.557 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:54:46.557 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:54:46.557 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:54:46.557 INFO:teuthology.orchestra.run.smithi198.stdout:Install 1 Package 2021-11-11T20:54:46.557 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:54:46.558 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 93 k 2021-11-11T20:54:46.558 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 281 k 2021-11-11T20:54:46.558 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:54:46.581 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 80 kB/s | 2.4 kB 00:00 2021-11-11T20:54:46.892 INFO:teuthology.orchestra.run.smithi198.stdout:rbd-fuse-17.0.0-8977.ga3a8018b.el8.x86_64.rpm 277 kB/s | 93 kB 00:00 2021-11-11T20:54:46.893 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:54:46.893 INFO:teuthology.orchestra.run.smithi198.stdout:Total 276 kB/s | 93 kB 00:00 2021-11-11T20:54:46.894 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:54:46.905 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:54:46.906 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:54:46.963 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:54:46.967 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:54:47.086 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:54:47.240 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : rbd-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:54:47.468 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: rbd-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:54:47.700 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : rbd-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:54:47.700 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:54:47.997 INFO:teuthology.orchestra.run.smithi096.stdout:Package python3-cephfs-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:48.448 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:54:48.449 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2021-11-11T20:54:48.449 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:54:48.550 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install python3-rbd 2021-11-11T20:54:48.887 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:49.402 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 92 kB/s | 2.8 kB 00:00 2021-11-11T20:54:49.584 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 80 kB/s | 2.4 kB 00:00 2021-11-11T20:54:50.980 INFO:teuthology.orchestra.run.smithi096.stdout:Package python3-rbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 is already installed. 2021-11-11T20:54:51.417 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:54:51.418 INFO:teuthology.orchestra.run.smithi096.stdout:Nothing to do. 2021-11-11T20:54:51.418 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:54:51.525 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install rbd-fuse 2021-11-11T20:54:51.860 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:54:52.358 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 90 kB/s | 2.8 kB 00:00 2021-11-11T20:54:52.542 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 76 kB/s | 2.4 kB 00:00 2021-11-11T20:54:54.394 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:54:54.395 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:54:54.395 INFO:teuthology.orchestra.run.smithi096.stdout: Package Architecture Version Repository Size 2021-11-11T20:54:54.395 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:54:54.395 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:54:54.396 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-fuse x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 93 k 2021-11-11T20:54:54.396 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:54:54.396 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:54:54.396 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:54:54.397 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2021-11-11T20:54:54.397 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:54:54.397 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 93 k 2021-11-11T20:54:54.397 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 281 k 2021-11-11T20:54:54.398 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:54:54.625 INFO:teuthology.orchestra.run.smithi096.stdout:rbd-fuse-17.0.0-8977.ga3a8018b.el8.x86_64.rpm 405 kB/s | 93 kB 00:00 2021-11-11T20:54:54.626 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:54:54.626 INFO:teuthology.orchestra.run.smithi096.stdout:Total 403 kB/s | 93 kB 00:00 2021-11-11T20:54:54.627 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:54:54.638 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:54:54.639 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:54:54.693 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:54:54.697 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:54:54.815 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:54:55.013 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : rbd-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:54:55.228 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: rbd-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:54:55.491 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : rbd-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:54:55.491 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:54:59.441 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:54:06 PM UTC. 2021-11-11T20:55:02.225 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:55:02.226 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:55:02.226 INFO:teuthology.orchestra.run.smithi198.stdout: rbd-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:55:02.226 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:55:02.226 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:55:02.354 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install rbd-mirror 2021-11-11T20:55:02.690 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:55:03.222 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 78 kB/s | 2.4 kB 00:00 2021-11-11T20:55:07.180 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:54:13 PM UTC. 2021-11-11T20:55:08.490 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 96 kB/s | 2.8 kB 00:00 2021-11-11T20:55:09.925 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:55:09.925 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:55:09.926 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-fuse-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:55:09.926 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:55:09.926 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:55:10.048 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install rbd-mirror 2021-11-11T20:55:10.383 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:55:10.895 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 97 kB/s | 2.8 kB 00:00 2021-11-11T20:55:13.604 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:55:15.912 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 78 kB/s | 2.4 kB 00:00 2021-11-11T20:55:16.200 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:55:16.201 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:55:16.201 INFO:teuthology.orchestra.run.smithi198.stdout: Package Arch Version Repo Size 2021-11-11T20:55:16.201 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:55:16.202 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:55:16.202 INFO:teuthology.orchestra.run.smithi198.stdout: rbd-mirror x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 3.8 M 2021-11-11T20:55:16.202 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:55:16.202 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:55:16.203 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:55:16.203 INFO:teuthology.orchestra.run.smithi198.stdout:Install 1 Package 2021-11-11T20:55:16.203 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:55:16.203 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 3.8 M 2021-11-11T20:55:16.204 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 19 M 2021-11-11T20:55:16.204 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:55:17.448 INFO:teuthology.orchestra.run.smithi198.stdout:rbd-mirror-17.0.0-8977.ga3a8018b.el8.x86_64.rpm 3.0 MB/s | 3.8 MB 00:01 2021-11-11T20:55:17.449 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:55:17.450 INFO:teuthology.orchestra.run.smithi198.stdout:Total 3.0 MB/s | 3.8 MB 00:01 2021-11-11T20:55:17.450 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:55:17.466 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:55:17.466 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:55:17.536 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:55:17.540 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:55:18.043 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:55:18.238 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : rbd-mirror-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:18.615 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: rbd-mirror-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:18.840 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : rbd-mirror-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:18.841 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:55:21.769 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:55:24.370 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:55:24.371 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:55:24.371 INFO:teuthology.orchestra.run.smithi096.stdout: Package Arch Version Repo Size 2021-11-11T20:55:24.371 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:55:24.371 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:55:24.372 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-mirror x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 3.8 M 2021-11-11T20:55:24.372 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:55:24.372 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:55:24.372 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:55:24.373 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2021-11-11T20:55:24.373 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:55:24.373 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 3.8 M 2021-11-11T20:55:24.373 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 19 M 2021-11-11T20:55:24.374 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:55:26.201 INFO:teuthology.orchestra.run.smithi096.stdout:rbd-mirror-17.0.0-8977.ga3a8018b.el8.x86_64.rpm 2.1 MB/s | 3.8 MB 00:01 2021-11-11T20:55:26.203 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:55:26.204 INFO:teuthology.orchestra.run.smithi096.stdout:Total 2.1 MB/s | 3.8 MB 00:01 2021-11-11T20:55:26.204 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:55:26.219 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:55:26.219 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:55:26.290 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:55:26.294 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:55:26.806 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:55:27.034 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : rbd-mirror-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:27.403 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: rbd-mirror-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:27.643 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : rbd-mirror-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:27.644 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:55:30.819 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:54:53 PM UTC. 2021-11-11T20:55:33.528 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:55:33.529 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:55:33.529 INFO:teuthology.orchestra.run.smithi198.stdout: rbd-mirror-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:55:33.529 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:55:33.529 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:55:33.659 DEBUG:teuthology.orchestra.run.smithi198:> sudo yum -y install rbd-nbd 2021-11-11T20:55:33.999 INFO:teuthology.orchestra.run.smithi198.stdout:Updating Subscription Management repositories. 2021-11-11T20:55:34.546 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 83 kB/s | 2.4 kB 00:00 2021-11-11T20:55:39.530 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:55:01 PM UTC. 2021-11-11T20:55:39.920 INFO:teuthology.orchestra.run.smithi198.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 97 kB/s | 2.8 kB 00:00 2021-11-11T20:55:42.258 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:55:42.258 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:55:42.258 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-mirror-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:55:42.259 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:55:42.259 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:55:42.384 DEBUG:teuthology.orchestra.run.smithi096:> sudo yum -y install rbd-nbd 2021-11-11T20:55:42.721 INFO:teuthology.orchestra.run.smithi096.stdout:Updating Subscription Management repositories. 2021-11-11T20:55:43.223 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 96 kB/s | 2.8 kB 00:00 2021-11-11T20:55:44.931 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:55:47.543 INFO:teuthology.orchestra.run.smithi198.stdout:Dependencies resolved. 2021-11-11T20:55:47.544 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:55:47.544 INFO:teuthology.orchestra.run.smithi198.stdout: Package Architecture Version Repository Size 2021-11-11T20:55:47.544 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:55:47.545 INFO:teuthology.orchestra.run.smithi198.stdout:Installing: 2021-11-11T20:55:47.545 INFO:teuthology.orchestra.run.smithi198.stdout: rbd-nbd x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 175 k 2021-11-11T20:55:47.545 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:55:47.545 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction Summary 2021-11-11T20:55:47.545 INFO:teuthology.orchestra.run.smithi198.stdout:================================================================================ 2021-11-11T20:55:47.546 INFO:teuthology.orchestra.run.smithi198.stdout:Install 1 Package 2021-11-11T20:55:47.546 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:55:47.546 INFO:teuthology.orchestra.run.smithi198.stdout:Total download size: 175 k 2021-11-11T20:55:47.546 INFO:teuthology.orchestra.run.smithi198.stdout:Installed size: 568 k 2021-11-11T20:55:47.547 INFO:teuthology.orchestra.run.smithi198.stdout:Downloading Packages: 2021-11-11T20:55:47.787 INFO:teuthology.orchestra.run.smithi198.stdout:rbd-nbd-17.0.0-8977.ga3a8018b.el8.x86_64.rpm 722 kB/s | 175 kB 00:00 2021-11-11T20:55:47.788 INFO:teuthology.orchestra.run.smithi198.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:55:47.788 INFO:teuthology.orchestra.run.smithi198.stdout:Total 718 kB/s | 175 kB 00:00 2021-11-11T20:55:47.789 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction check 2021-11-11T20:55:47.801 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction check succeeded. 2021-11-11T20:55:47.801 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction test 2021-11-11T20:55:47.856 INFO:teuthology.orchestra.run.smithi198.stdout:Transaction test succeeded. 2021-11-11T20:55:47.860 INFO:teuthology.orchestra.run.smithi198.stdout:Running transaction 2021-11-11T20:55:47.984 INFO:teuthology.orchestra.run.smithi198.stdout: Preparing : 1/1 2021-11-11T20:55:48.180 INFO:teuthology.orchestra.run.smithi198.stdout: Installing : rbd-nbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:48.209 INFO:teuthology.orchestra.run.smithi096.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 81 kB/s | 2.4 kB 00:00 2021-11-11T20:55:48.411 INFO:teuthology.orchestra.run.smithi198.stdout: Running scriptlet: rbd-nbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:48.640 INFO:teuthology.orchestra.run.smithi198.stdout: Verifying : rbd-nbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:48.641 INFO:teuthology.orchestra.run.smithi198.stdout:Installed products updated. 2021-11-11T20:55:53.583 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:46:12 PM UTC. 2021-11-11T20:55:56.184 INFO:teuthology.orchestra.run.smithi096.stdout:Dependencies resolved. 2021-11-11T20:55:56.185 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:55:56.186 INFO:teuthology.orchestra.run.smithi096.stdout: Package Architecture Version Repository Size 2021-11-11T20:55:56.186 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:55:56.186 INFO:teuthology.orchestra.run.smithi096.stdout:Installing: 2021-11-11T20:55:56.186 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-nbd x86_64 2:17.0.0-8977.ga3a8018b.el8 ceph 175 k 2021-11-11T20:55:56.187 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:55:56.187 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction Summary 2021-11-11T20:55:56.187 INFO:teuthology.orchestra.run.smithi096.stdout:================================================================================ 2021-11-11T20:55:56.187 INFO:teuthology.orchestra.run.smithi096.stdout:Install 1 Package 2021-11-11T20:55:56.188 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:55:56.188 INFO:teuthology.orchestra.run.smithi096.stdout:Total download size: 175 k 2021-11-11T20:55:56.188 INFO:teuthology.orchestra.run.smithi096.stdout:Installed size: 568 k 2021-11-11T20:55:56.188 INFO:teuthology.orchestra.run.smithi096.stdout:Downloading Packages: 2021-11-11T20:55:56.454 INFO:teuthology.orchestra.run.smithi096.stdout:rbd-nbd-17.0.0-8977.ga3a8018b.el8.x86_64.rpm 655 kB/s | 175 kB 00:00 2021-11-11T20:55:56.454 INFO:teuthology.orchestra.run.smithi096.stdout:-------------------------------------------------------------------------------- 2021-11-11T20:55:56.455 INFO:teuthology.orchestra.run.smithi096.stdout:Total 652 kB/s | 175 kB 00:00 2021-11-11T20:55:56.455 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction check 2021-11-11T20:55:56.468 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction check succeeded. 2021-11-11T20:55:56.468 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction test 2021-11-11T20:55:56.524 INFO:teuthology.orchestra.run.smithi096.stdout:Transaction test succeeded. 2021-11-11T20:55:56.528 INFO:teuthology.orchestra.run.smithi096.stdout:Running transaction 2021-11-11T20:55:56.651 INFO:teuthology.orchestra.run.smithi096.stdout: Preparing : 1/1 2021-11-11T20:55:56.858 INFO:teuthology.orchestra.run.smithi096.stdout: Installing : rbd-nbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:57.085 INFO:teuthology.orchestra.run.smithi096.stdout: Running scriptlet: rbd-nbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:57.343 INFO:teuthology.orchestra.run.smithi096.stdout: Verifying : rbd-nbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 1/1 2021-11-11T20:55:57.343 INFO:teuthology.orchestra.run.smithi096.stdout:Installed products updated. 2021-11-11T20:56:00.604 INFO:teuthology.orchestra.run.smithi198.stdout:Last metadata expiration check: 0:00:05 ago on Thu 11 Nov 2021 08:55:25 PM UTC. 2021-11-11T20:56:03.340 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:03.340 INFO:teuthology.orchestra.run.smithi198.stdout:Installed: 2021-11-11T20:56:03.341 INFO:teuthology.orchestra.run.smithi198.stdout: rbd-nbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:56:03.341 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:03.341 INFO:teuthology.orchestra.run.smithi198.stdout:Complete! 2021-11-11T20:56:03.466 DEBUG:teuthology.parallel:result is None 2021-11-11T20:56:09.124 INFO:teuthology.orchestra.run.smithi096.stdout:Last metadata expiration check: 0:00:06 ago on Thu 11 Nov 2021 08:55:33 PM UTC. 2021-11-11T20:56:11.803 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:11.804 INFO:teuthology.orchestra.run.smithi096.stdout:Installed: 2021-11-11T20:56:11.804 INFO:teuthology.orchestra.run.smithi096.stdout: rbd-nbd-2:17.0.0-8977.ga3a8018b.el8.x86_64 2021-11-11T20:56:11.804 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:11.804 INFO:teuthology.orchestra.run.smithi096.stdout:Complete! 2021-11-11T20:56:11.955 DEBUG:teuthology.parallel:result is None 2021-11-11T20:56:11.956 DEBUG:teuthology.orchestra.run.smithi096:> sudo lsb_release -is 2021-11-11T20:56:11.995 INFO:teuthology.orchestra.run.smithi096.stdout:RedHatEnterprise 2021-11-11T20:56:11.996 DEBUG:teuthology.misc:System to be installed: RedHatEnterprise 2021-11-11T20:56:11.997 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=a3a8018ba3cfe536252b397411bf18b7828c6098 2021-11-11T20:56:12.212 DEBUG:teuthology.orchestra.run.smithi096:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2021-11-11T20:56:12.232 INFO:teuthology.orchestra.run.smithi096.stdout:17.0.0-8977.ga3a8018b.el8 2021-11-11T20:56:12.233 INFO:teuthology.packaging:The installed version of ceph is 17.0.0-8977.ga3a8018b.el8 2021-11-11T20:56:12.233 INFO:teuthology.task.install:The correct ceph version 17.0.0-8977.ga3a8018b is installed. 2021-11-11T20:56:12.234 DEBUG:teuthology.orchestra.run.smithi198:> sudo lsb_release -is 2021-11-11T20:56:12.280 INFO:teuthology.orchestra.run.smithi198.stdout:RedHatEnterprise 2021-11-11T20:56:12.281 DEBUG:teuthology.misc:System to be installed: RedHatEnterprise 2021-11-11T20:56:12.282 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=a3a8018ba3cfe536252b397411bf18b7828c6098 2021-11-11T20:56:12.490 DEBUG:teuthology.orchestra.run.smithi198:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2021-11-11T20:56:12.514 INFO:teuthology.orchestra.run.smithi198.stdout:17.0.0-8977.ga3a8018b.el8 2021-11-11T20:56:12.514 INFO:teuthology.packaging:The installed version of ceph is 17.0.0-8977.ga3a8018b.el8 2021-11-11T20:56:12.514 INFO:teuthology.task.install:The correct ceph version 17.0.0-8977.ga3a8018b is installed. 2021-11-11T20:56:12.515 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2021-11-11T20:56:12.515 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:12.516 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2021-11-11T20:56:12.541 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:12.542 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2021-11-11T20:56:12.581 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2021-11-11T20:56:12.581 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:12.582 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/usr/bin/daemon-helper 2021-11-11T20:56:12.607 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod a=rx -- /usr/bin/daemon-helper 2021-11-11T20:56:12.671 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:12.672 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd of=/usr/bin/daemon-helper 2021-11-11T20:56:12.699 DEBUG:teuthology.orchestra.run.smithi198:> sudo chmod a=rx -- /usr/bin/daemon-helper 2021-11-11T20:56:12.766 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2021-11-11T20:56:12.766 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:12.767 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/usr/bin/adjust-ulimits 2021-11-11T20:56:12.791 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2021-11-11T20:56:12.855 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:12.855 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd of=/usr/bin/adjust-ulimits 2021-11-11T20:56:12.881 DEBUG:teuthology.orchestra.run.smithi198:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2021-11-11T20:56:12.946 INFO:teuthology.run_tasks:Running task ceph... 2021-11-11T20:56:13.017 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2021-11-11T20:56:13.018 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod 777 /var/log/ceph 2021-11-11T20:56:13.021 DEBUG:teuthology.orchestra.run.smithi198:> sudo chmod 777 /var/log/ceph 2021-11-11T20:56:13.045 INFO:tasks.ceph:Disabling ceph logrotate... 2021-11-11T20:56:13.046 DEBUG:teuthology.orchestra.run.smithi096:> sudo rm -f -- /etc/logrotate.d/ceph 2021-11-11T20:56:13.084 DEBUG:teuthology.orchestra.run.smithi198:> sudo rm -f -- /etc/logrotate.d/ceph 2021-11-11T20:56:13.110 INFO:tasks.ceph:Creating extra log directories... 2021-11-11T20:56:13.111 DEBUG:teuthology.orchestra.run.smithi096:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2021-11-11T20:56:13.149 DEBUG:teuthology.orchestra.run.smithi198:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2021-11-11T20:56:13.185 INFO:tasks.ceph:Creating ceph cluster c1... 2021-11-11T20:56:13.185 INFO:tasks.ceph:config {'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug filestore': 20, 'debug journal': 20, 'debug ms': 20, 'debug osd': 25, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)'], 'cpu_profile': set(), 'cluster': 'c1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2021-11-11T20:56:13.185 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/soumyakoduri-2021-11-11_20:19:05-rgw-cloudtiering-distro-basic-smithi/6497986', 'branch': 'cloudtiering', 'description': 'rgw/multisite/{clusters frontend/beast ignore-pg-availability omap_limits overrides realms/three-zone-plus-pubsub supported-random-distro$/{rhel_8} tasks/test_multi}', 'email': None, 'first_in_suite': False, 'job_id': '6497986', 'kernel': {'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'name': 'soumyakoduri-2021-11-11_20:19:05-rgw-cloudtiering-distro-basic-smithi', 'nuke-on-error': True, 'os_type': 'rhel', 'os_version': '8.4', 'overrides': {'admin_socket': {'branch': 'cloudtiering'}, 'ceph': {'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug filestore': 20, 'debug journal': 20, 'debug ms': 20, 'debug osd': 25, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)'], 'log-whitelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098', 'wait-for-scrub': False}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {'osd default pool size': 2}}}, 'install': {'ceph': {'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098'}}, 'rgw': {'compression type': 'random', 'frontend': 'beast'}, 'rgw-multisite': {'realm': {'is default': True, 'name': 'test-realm'}, 'zonegroups': [{'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'test-zonegroup', 'zones': [{'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'test-zone1'}, {'endpoints': ['c2.client.0'], 'is_default': True, 'name': 'test-zone2'}, {'endpoints': ['c1.client.1'], 'name': 'test-zone3'}, {'endpoints': ['c2.client.1'], 'is_pubsub': True, 'name': 'test-zone4'}]}]}, 'rgw-multisite-tests': {'args': ['tests.py', 'tests_ps.py']}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'cloudtiering', 'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098'}}, 'owner': 'scheduled_soumyakoduri@teuthology', 'priority': 100, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['c1.mon.a', 'c1.mgr.x', 'c1.osd.0', 'c1.osd.1', 'c1.osd.2', 'c1.client.0', 'c1.client.1'], ['c2.mon.a', 'c2.mgr.x', 'c2.osd.0', 'c2.osd.1', 'c2.osd.2', 'c2.client.0', 'c2.client.1']], 'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098', 'sleep_before_teardown': 0, 'suite': 'rgw', 'suite_branch': 'cloudtiering', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph-c_a3a8018ba3cfe536252b397411bf18b7828c6098/qa', 'suite_relpath': 'qa', 'suite_repo': 'git://git.ceph.com/ceph-ci.git', 'suite_sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098', 'targets': {'smithi096.front.sepia.ceph.com': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCnnzgfZwb9DMmAqSFD0JEkotGtIng9gnT9CmKOgT3btn8RwZTgGqnrdK9xImGAZB25OR1Jma+qf1JWFmOpKBb+PipPVc6yGHbnCX0CLNS4vhkTTvQuBCxnu6Q6MOru6+5yzF3S3PK/bP24It0Crezuq6il9HCWMOf4m7AVYWp867OSYY+x29gpa58Ag2JcBysrC+LPPT0q0k60P6QGLgsp0mIxDEQUfx5e85+dWseIvmN5VzbOeSzuDMRuMr0GcqqTDJSnPI87JzequKN8CrYN6uXtyKZgvj/or7v1ycHdIWFn5MYz/eWRvv5Z5I0mlt8mSLr8U889gsVhH9kwe/e4I/DRdwjBY4WLRbtQe2YjTsTocusQdUwrHwSHuDyvQ/xj/GzvM7OxCI8E8Tq9n2bNWVB33r647D+QJxe42FioRqbdW8QBMAHA2ciBPK6bJ4iuL8AvtPJeD4GhoWgxpZRnrzgu0Xm7+MzcDJeoJ+3ygOrTHprj6AHz+0JMZoFtZEs=', 'smithi198.front.sepia.ceph.com': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9jvdzDl2KeYL6DgYVb9e9leVYHjq3W9lqcBtDftr7EXr6wf0lNmnP+FikYaJOKq8PtBzr50l1o4lYLSSUhsZKS7GwWH1E1T95iY9FYTjv6M3LamGEygDOl905Y1OkkTmcrSWndwQIFh7yJnUVGUzhJxeDMICBoBclIsXjxAHYm4EYaePhsyZ3P6zAhIld/fpRSR3xgN5ggGmryOIHi+GHnN7qTYv+Qso1/vRKIcnFw7Q8GbRbx0wSb6sLwCEL4sA5O4CfLMmVsomQeD8wU8ze9voK/060tkeQtOo77ZTC8UEavnElU4Q/p85M7RQdiIf/lcJh7yN8NkieVh019SiiLPkgZ8D+rtrQsiV6vW5zv3VfC38cIi+l8rilwYomn/r4u6CJc0Og1bIp9yZLcS30RsoShNFxTLNjfBKYNz9IyFFfxtTZyDcPtGwxuZgQyfRhmhJJSwZlOMM7zjbHUB15z/l86+/pT9NYWX0WmHuwRXlQzUwAky053tlXYH0Ui4M='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': None}, {'ceph': {'cluster': 'c1', 'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug filestore': 20, 'debug journal': 20, 'debug ms': 20, 'debug osd': 25, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)'], 'log-whitelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098', 'wait-for-scrub': False}}, {'ceph': {'cluster': 'c2'}}, {'rgw': {'c1.client.0': {'port': 8000}, 'c1.client.1': {'port': 8001}, 'c2.client.0': {'port': 8000}, 'c2.client.1': {'port': 8001}}}, {'rgw-multisite': None}, {'rgw-multisite-tests': {'config': {'reconfigure_delay': 60}}}], 'teuthology_branch': 'master', 'timestamp': '2021-11-11_20:19:05', 'tube': 'smithi', 'user': 'soumyakoduri', 'verbose': True, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.20261'} 2021-11-11T20:56:13.186 DEBUG:teuthology.orchestra.run.smithi096:> install -d -m0755 -- /home/ubuntu/cephtest/c1.data 2021-11-11T20:56:13.213 DEBUG:teuthology.orchestra.run.smithi198:> install -d -m0755 -- /home/ubuntu/cephtest/c1.data 2021-11-11T20:56:13.229 DEBUG:teuthology.orchestra.run.smithi096:> sudo install -d -m0777 -- /var/run/ceph 2021-11-11T20:56:13.267 DEBUG:teuthology.orchestra.run.smithi198:> sudo install -d -m0777 -- /var/run/ceph 2021-11-11T20:56:13.292 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:13.292 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/scratch_devs of=/dev/stdout 2021-11-11T20:56:13.342 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2021-11-11T20:56:13.342 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_1 && sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_1 2021-11-11T20:56:13.395 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2021-11-11T20:56:13.396 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2021-11-11T20:56:13.396 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 6h/6d Inode: 118973 Links: 1 2021-11-11T20:56:13.396 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2021-11-11T20:56:13.397 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2021-11-11T20:56:13.397 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2021-11-11 20:55:27.218916172 +0000 2021-11-11T20:56:13.397 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2021-11-11 20:45:09.066589280 +0000 2021-11-11T20:56:13.397 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2021-11-11 20:45:09.066589280 +0000 2021-11-11T20:56:13.398 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: - 2021-11-11T20:56:13.405 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2021-11-11T20:56:13.405 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2021-11-11T20:56:13.405 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.000214229 s, 2.4 MB/s 2021-11-11T20:56:13.408 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_2 && sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_2 2021-11-11T20:56:13.462 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2021-11-11T20:56:13.462 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2021-11-11T20:56:13.463 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 6h/6d Inode: 114314 Links: 1 2021-11-11T20:56:13.463 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2021-11-11T20:56:13.463 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2021-11-11T20:56:13.463 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2021-11-11 20:55:27.218916172 +0000 2021-11-11T20:56:13.464 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2021-11-11 20:45:09.431580751 +0000 2021-11-11T20:56:13.464 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2021-11-11 20:45:09.431580751 +0000 2021-11-11T20:56:13.464 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: - 2021-11-11T20:56:13.471 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2021-11-11T20:56:13.472 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2021-11-11T20:56:13.472 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.000211331 s, 2.4 MB/s 2021-11-11T20:56:13.474 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_3 && sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_3 2021-11-11T20:56:13.528 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2021-11-11T20:56:13.529 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2021-11-11T20:56:13.529 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 6h/6d Inode: 115045 Links: 1 2021-11-11T20:56:13.529 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2021-11-11T20:56:13.529 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2021-11-11T20:56:13.530 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2021-11-11 20:55:27.218916172 +0000 2021-11-11T20:56:13.530 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2021-11-11 20:45:09.756573156 +0000 2021-11-11T20:56:13.530 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2021-11-11 20:45:09.756573156 +0000 2021-11-11T20:56:13.530 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: - 2021-11-11T20:56:13.538 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2021-11-11T20:56:13.538 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2021-11-11T20:56:13.538 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.000207684 s, 2.5 MB/s 2021-11-11T20:56:13.540 DEBUG:teuthology.orchestra.run.smithi096:> stat /dev/vg_nvme/lv_4 && sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_4 2021-11-11T20:56:13.595 INFO:teuthology.orchestra.run.smithi096.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2021-11-11T20:56:13.595 INFO:teuthology.orchestra.run.smithi096.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2021-11-11T20:56:13.596 INFO:teuthology.orchestra.run.smithi096.stdout:Device: 6h/6d Inode: 116098 Links: 1 2021-11-11T20:56:13.596 INFO:teuthology.orchestra.run.smithi096.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2021-11-11T20:56:13.596 INFO:teuthology.orchestra.run.smithi096.stdout:Context: system_u:object_r:device_t:s0 2021-11-11T20:56:13.596 INFO:teuthology.orchestra.run.smithi096.stdout:Access: 2021-11-11 20:55:27.218916172 +0000 2021-11-11T20:56:13.596 INFO:teuthology.orchestra.run.smithi096.stdout:Modify: 2021-11-11 20:45:10.111564860 +0000 2021-11-11T20:56:13.597 INFO:teuthology.orchestra.run.smithi096.stdout:Change: 2021-11-11 20:45:10.111564860 +0000 2021-11-11T20:56:13.597 INFO:teuthology.orchestra.run.smithi096.stdout: Birth: - 2021-11-11T20:56:13.604 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records in 2021-11-11T20:56:13.605 INFO:teuthology.orchestra.run.smithi096.stderr:1+0 records out 2021-11-11T20:56:13.605 INFO:teuthology.orchestra.run.smithi096.stderr:512 bytes copied, 0.000205946 s, 2.5 MB/s 2021-11-11T20:56:13.607 INFO:tasks.ceph:osd dev map: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2021-11-11T20:56:13.608 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi096.front.sepia.ceph.com'): {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'}} 2021-11-11T20:56:13.608 INFO:tasks.ceph:Generating config... 2021-11-11T20:56:13.609 INFO:tasks.ceph:[client] debug rgw = 20 2021-11-11T20:56:13.610 INFO:tasks.ceph:[client] rgw crypt require ssl = False 2021-11-11T20:56:13.610 INFO:tasks.ceph:[client] rgw crypt s3 kms backend = testing 2021-11-11T20:56:13.610 INFO:tasks.ceph:[client] rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2021-11-11T20:56:13.611 INFO:tasks.ceph:[client] rgw curl low speed time = 300 2021-11-11T20:56:13.611 INFO:tasks.ceph:[client] rgw data log num shards = 4 2021-11-11T20:56:13.611 INFO:tasks.ceph:[client] rgw md log max shards = 4 2021-11-11T20:56:13.612 INFO:tasks.ceph:[client] rgw sync log trim interval = 0 2021-11-11T20:56:13.612 INFO:tasks.ceph:[client] rgw sync meta inject err probability = 0.1 2021-11-11T20:56:13.612 INFO:tasks.ceph:[client] rgw sync obj etag verify = True 2021-11-11T20:56:13.613 INFO:tasks.ceph:[client] setgroup = ceph 2021-11-11T20:56:13.613 INFO:tasks.ceph:[client] setuser = ceph 2021-11-11T20:56:13.613 INFO:tasks.ceph:[mgr] debug mgr = 20 2021-11-11T20:56:13.613 INFO:tasks.ceph:[mgr] debug ms = 1 2021-11-11T20:56:13.614 INFO:tasks.ceph:[mon] debug mon = 20 2021-11-11T20:56:13.614 INFO:tasks.ceph:[mon] debug ms = 1 2021-11-11T20:56:13.614 INFO:tasks.ceph:[mon] debug paxos = 20 2021-11-11T20:56:13.615 INFO:tasks.ceph:[osd] debug filestore = 20 2021-11-11T20:56:13.615 INFO:tasks.ceph:[osd] debug journal = 20 2021-11-11T20:56:13.615 INFO:tasks.ceph:[osd] debug ms = 20 2021-11-11T20:56:13.616 INFO:tasks.ceph:[osd] debug osd = 25 2021-11-11T20:56:13.616 INFO:tasks.ceph:[osd] rocksdb delete range threshold = 0 2021-11-11T20:56:13.616 INFO:tasks.ceph:[osd.0] osd_max_omap_entries_per_request = 10 2021-11-11T20:56:13.617 INFO:tasks.ceph:[osd.1] osd_max_omap_entries_per_request = 1000 2021-11-11T20:56:13.617 INFO:tasks.ceph:[osd.2] osd_max_omap_entries_per_request = 10000 2021-11-11T20:56:13.617 INFO:tasks.ceph:Setting up c1.mon.a... 2021-11-11T20:56:13.618 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/c1.keyring 2021-11-11T20:56:13.700 INFO:teuthology.orchestra.run.smithi096.stdout:creating /etc/ceph/c1.keyring 2021-11-11T20:56:13.702 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/c1.keyring 2021-11-11T20:56:13.739 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod 0644 /etc/ceph/c1.keyring 2021-11-11T20:56:13.803 DEBUG:tasks.ceph:Ceph mon addresses: [('c1.mon.a', '172.21.15.96')] 2021-11-11T20:56:13.804 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.96'}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'debug filestore': 20, 'debug journal': 20, 'debug ms': 20, 'debug osd': 25, 'rocksdb delete range threshold': 0}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mon.a': {}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}} 2021-11-11T20:56:13.805 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:13.805 DEBUG:teuthology.orchestra.run.smithi096:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2021-11-11T20:56:13.858 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.96 --print /home/ubuntu/cephtest/c1.monmap 2021-11-11T20:56:13.931 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:56:13.932 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:56:13.933 INFO:teuthology.orchestra.run.smithi096.stdout:monmaptool: monmap file /home/ubuntu/cephtest/c1.monmap 2021-11-11T20:56:13.933 INFO:teuthology.orchestra.run.smithi096.stdout:monmaptool: generated fsid 472dfe13-0d20-45f4-8fbc-1a0e45d2ce49 2021-11-11T20:56:13.934 INFO:teuthology.orchestra.run.smithi096.stdout:setting min_mon_release = octopus 2021-11-11T20:56:13.934 INFO:teuthology.orchestra.run.smithi096.stdout:epoch 0 2021-11-11T20:56:13.934 INFO:teuthology.orchestra.run.smithi096.stdout:fsid 472dfe13-0d20-45f4-8fbc-1a0e45d2ce49 2021-11-11T20:56:13.934 INFO:teuthology.orchestra.run.smithi096.stdout:last_changed 2021-11-11T20:56:13.933318+0000 2021-11-11T20:56:13.935 INFO:teuthology.orchestra.run.smithi096.stdout:created 2021-11-11T20:56:13.933318+0000 2021-11-11T20:56:13.935 INFO:teuthology.orchestra.run.smithi096.stdout:min_mon_release 15 (octopus) 2021-11-11T20:56:13.935 INFO:teuthology.orchestra.run.smithi096.stdout:election_strategy: 1 2021-11-11T20:56:13.935 INFO:teuthology.orchestra.run.smithi096.stdout:0: [v2:172.21.15.96:3300/0,v1:172.21.15.96:6789/0] mon.a 2021-11-11T20:56:13.935 INFO:teuthology.orchestra.run.smithi096.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/c1.monmap (1 monitors) 2021-11-11T20:56:13.936 DEBUG:teuthology.orchestra.run.smithi096:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2021-11-11T20:56:13.987 INFO:tasks.ceph:Writing /etc/ceph/c1.conf for FSID 472dfe13-0d20-45f4-8fbc-1a0e45d2ce49... 2021-11-11T20:56:13.988 DEBUG:teuthology.orchestra.run.smithi096:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c1.conf && sudo chmod 0644 /etc/ceph/c1.conf > /dev/null 2021-11-11T20:56:14.029 DEBUG:teuthology.orchestra.run.smithi198:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c1.conf && sudo chmod 0644 /etc/ceph/c1.conf > /dev/null 2021-11-11T20:56:14.067 INFO:teuthology.orchestra.run.smithi096.stdout:[global] 2021-11-11T20:56:14.068 INFO:teuthology.orchestra.run.smithi096.stdout: chdir = "" 2021-11-11T20:56:14.068 INFO:teuthology.orchestra.run.smithi096.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2021-11-11T20:56:14.068 INFO:teuthology.orchestra.run.smithi096.stdout: auth supported = cephx 2021-11-11T20:56:14.069 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.069 INFO:teuthology.orchestra.run.smithi096.stdout: filestore xattr use omap = true 2021-11-11T20:56:14.069 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.069 INFO:teuthology.orchestra.run.smithi096.stdout: mon clock drift allowed = 1.000 2021-11-11T20:56:14.069 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.070 INFO:teuthology.orchestra.run.smithi096.stdout: osd crush chooseleaf type = 0 2021-11-11T20:56:14.070 INFO:teuthology.orchestra.run.smithi096.stdout: auth debug = true 2021-11-11T20:56:14.070 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.070 INFO:teuthology.orchestra.run.smithi096.stdout: ms die on old message = true 2021-11-11T20:56:14.071 INFO:teuthology.orchestra.run.smithi096.stdout: ms die on bug = true 2021-11-11T20:56:14.071 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.071 INFO:teuthology.orchestra.run.smithi096.stdout: mon max pg per osd = 10000 # >= luminous 2021-11-11T20:56:14.071 INFO:teuthology.orchestra.run.smithi096.stdout: mon pg warn max object skew = 0 2021-11-11T20:56:14.072 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.072 INFO:teuthology.orchestra.run.smithi096.stdout: # disable pg_autoscaler by default for new pools 2021-11-11T20:56:14.072 INFO:teuthology.orchestra.run.smithi096.stdout: osd_pool_default_pg_autoscale_mode = off 2021-11-11T20:56:14.072 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.073 INFO:teuthology.orchestra.run.smithi096.stdout: osd pool default size = 2 2021-11-11T20:56:14.073 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.073 INFO:teuthology.orchestra.run.smithi096.stdout: mon osd allow primary affinity = true 2021-11-11T20:56:14.073 INFO:teuthology.orchestra.run.smithi096.stdout: mon osd allow pg remap = true 2021-11-11T20:56:14.074 INFO:teuthology.orchestra.run.smithi096.stdout: mon warn on legacy crush tunables = false 2021-11-11T20:56:14.074 INFO:teuthology.orchestra.run.smithi096.stdout: mon warn on crush straw calc version zero = false 2021-11-11T20:56:14.074 INFO:teuthology.orchestra.run.smithi096.stdout: mon warn on no sortbitwise = false 2021-11-11T20:56:14.074 INFO:teuthology.orchestra.run.smithi096.stdout: mon warn on osd down out interval zero = false 2021-11-11T20:56:14.075 INFO:teuthology.orchestra.run.smithi096.stdout: mon warn on too few osds = false 2021-11-11T20:56:14.075 INFO:teuthology.orchestra.run.smithi096.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2021-11-11T20:56:14.075 INFO:teuthology.orchestra.run.smithi096.stdout: mon_warn_on_pool_no_redundancy = false 2021-11-11T20:56:14.075 INFO:teuthology.orchestra.run.smithi096.stdout: mon_allow_pool_size_one = true 2021-11-11T20:56:14.075 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.076 INFO:teuthology.orchestra.run.smithi096.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2021-11-11T20:56:14.076 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.076 INFO:teuthology.orchestra.run.smithi096.stdout: osd default data pool replay window = 5 2021-11-11T20:56:14.076 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.077 INFO:teuthology.orchestra.run.smithi096.stdout: mon allow pool delete = true 2021-11-11T20:56:14.077 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.077 INFO:teuthology.orchestra.run.smithi096.stdout: mon cluster log file level = debug 2021-11-11T20:56:14.077 INFO:teuthology.orchestra.run.smithi096.stdout: debug asserts on shutdown = true 2021-11-11T20:56:14.078 INFO:teuthology.orchestra.run.smithi096.stdout: mon health detail to clog = false 2021-11-11T20:56:14.078 INFO:teuthology.orchestra.run.smithi096.stdout: mon host = 172.21.15.96 2021-11-11T20:56:14.078 INFO:teuthology.orchestra.run.smithi096.stdout: fsid = 472dfe13-0d20-45f4-8fbc-1a0e45d2ce49 2021-11-11T20:56:14.078 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.079 INFO:teuthology.orchestra.run.smithi096.stdout:[osd] 2021-11-11T20:56:14.079 INFO:teuthology.orchestra.run.smithi096.stdout: osd journal size = 100 2021-11-11T20:56:14.079 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.079 INFO:teuthology.orchestra.run.smithi096.stdout: osd scrub load threshold = 5.0 2021-11-11T20:56:14.080 INFO:teuthology.orchestra.run.smithi096.stdout: osd scrub max interval = 600 2021-11-11T20:56:14.080 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.080 INFO:teuthology.orchestra.run.smithi096.stdout: osd recover clone overlap = true 2021-11-11T20:56:14.080 INFO:teuthology.orchestra.run.smithi096.stdout: osd recovery max chunk = 1048576 2021-11-11T20:56:14.081 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.081 INFO:teuthology.orchestra.run.smithi096.stdout: osd debug shutdown = true 2021-11-11T20:56:14.081 INFO:teuthology.orchestra.run.smithi096.stdout: osd debug op order = true 2021-11-11T20:56:14.081 INFO:teuthology.orchestra.run.smithi096.stdout: osd debug verify stray on activate = true 2021-11-11T20:56:14.082 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.082 INFO:teuthology.orchestra.run.smithi096.stdout: osd open classes on start = true 2021-11-11T20:56:14.082 INFO:teuthology.orchestra.run.smithi096.stdout: osd debug pg log writeout = true 2021-11-11T20:56:14.082 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.083 INFO:teuthology.orchestra.run.smithi096.stdout: osd deep scrub update digest min age = 30 2021-11-11T20:56:14.083 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.083 INFO:teuthology.orchestra.run.smithi096.stdout: osd map max advance = 10 2021-11-11T20:56:14.083 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.083 INFO:teuthology.orchestra.run.smithi096.stdout: journal zero on create = true 2021-11-11T20:56:14.084 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.084 INFO:teuthology.orchestra.run.smithi096.stdout: filestore ondisk finisher threads = 3 2021-11-11T20:56:14.084 INFO:teuthology.orchestra.run.smithi096.stdout: filestore apply finisher threads = 3 2021-11-11T20:56:14.084 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.085 INFO:teuthology.orchestra.run.smithi096.stdout: bdev debug aio = true 2021-11-11T20:56:14.085 INFO:teuthology.orchestra.run.smithi096.stdout: osd debug misdirected ops = true 2021-11-11T20:56:14.085 INFO:teuthology.orchestra.run.smithi096.stdout: debug filestore = 20 2021-11-11T20:56:14.085 INFO:teuthology.orchestra.run.smithi096.stdout: debug journal = 20 2021-11-11T20:56:14.086 INFO:teuthology.orchestra.run.smithi096.stdout: debug ms = 20 2021-11-11T20:56:14.086 INFO:teuthology.orchestra.run.smithi096.stdout: debug osd = 25 2021-11-11T20:56:14.086 INFO:teuthology.orchestra.run.smithi096.stdout: rocksdb delete range threshold = 0 2021-11-11T20:56:14.086 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.087 INFO:teuthology.orchestra.run.smithi096.stdout:[mgr] 2021-11-11T20:56:14.087 INFO:teuthology.orchestra.run.smithi096.stdout: debug ms = 1 2021-11-11T20:56:14.087 INFO:teuthology.orchestra.run.smithi096.stdout: debug mgr = 20 2021-11-11T20:56:14.087 INFO:teuthology.orchestra.run.smithi096.stdout: debug mon = 20 2021-11-11T20:56:14.087 INFO:teuthology.orchestra.run.smithi096.stdout: debug auth = 20 2021-11-11T20:56:14.088 INFO:teuthology.orchestra.run.smithi096.stdout: mon reweight min pgs per osd = 4 2021-11-11T20:56:14.088 INFO:teuthology.orchestra.run.smithi096.stdout: mon reweight min bytes per osd = 10 2021-11-11T20:56:14.088 INFO:teuthology.orchestra.run.smithi096.stdout: mgr/telemetry/nag = false 2021-11-11T20:56:14.088 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.089 INFO:teuthology.orchestra.run.smithi096.stdout:[mon] 2021-11-11T20:56:14.089 INFO:teuthology.orchestra.run.smithi096.stdout: debug ms = 1 2021-11-11T20:56:14.089 INFO:teuthology.orchestra.run.smithi096.stdout: debug mon = 20 2021-11-11T20:56:14.089 INFO:teuthology.orchestra.run.smithi096.stdout: debug paxos = 20 2021-11-11T20:56:14.090 INFO:teuthology.orchestra.run.smithi096.stdout: debug auth = 20 2021-11-11T20:56:14.090 INFO:teuthology.orchestra.run.smithi096.stdout: mon data avail warn = 5 2021-11-11T20:56:14.090 INFO:teuthology.orchestra.run.smithi096.stdout: mon mgr mkfs grace = 240 2021-11-11T20:56:14.090 INFO:teuthology.orchestra.run.smithi096.stdout: mon reweight min pgs per osd = 4 2021-11-11T20:56:14.091 INFO:teuthology.orchestra.run.smithi096.stdout: mon osd reporter subtree level = osd 2021-11-11T20:56:14.091 INFO:teuthology.orchestra.run.smithi096.stdout: mon osd prime pg temp = true 2021-11-11T20:56:14.091 INFO:teuthology.orchestra.run.smithi096.stdout: mon reweight min bytes per osd = 10 2021-11-11T20:56:14.091 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.092 INFO:teuthology.orchestra.run.smithi096.stdout: # rotate auth tickets quickly to exercise renewal paths 2021-11-11T20:56:14.092 INFO:teuthology.orchestra.run.smithi096.stdout: auth mon ticket ttl = 660 # 11m 2021-11-11T20:56:14.092 INFO:teuthology.orchestra.run.smithi096.stdout: auth service ticket ttl = 240 # 4m 2021-11-11T20:56:14.092 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.092 INFO:teuthology.orchestra.run.smithi096.stdout: # don't complain about insecure global_id in the test suite 2021-11-11T20:56:14.093 INFO:teuthology.orchestra.run.smithi096.stdout: mon_warn_on_insecure_global_id_reclaim = false 2021-11-11T20:56:14.093 INFO:teuthology.orchestra.run.smithi096.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2021-11-11T20:56:14.093 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:14.093 INFO:teuthology.orchestra.run.smithi096.stdout:[client] 2021-11-11T20:56:14.094 INFO:teuthology.orchestra.run.smithi096.stdout: rgw cache enabled = true 2021-11-11T20:56:14.094 INFO:teuthology.orchestra.run.smithi096.stdout: rgw enable ops log = true 2021-11-11T20:56:14.094 INFO:teuthology.orchestra.run.smithi096.stdout: rgw enable usage log = true 2021-11-11T20:56:14.095 INFO:teuthology.orchestra.run.smithi096.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2021-11-11T20:56:14.095 INFO:teuthology.orchestra.run.smithi096.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2021-11-11T20:56:14.095 INFO:teuthology.orchestra.run.smithi096.stdout: debug rgw = 20 2021-11-11T20:56:14.095 INFO:teuthology.orchestra.run.smithi096.stdout: rgw crypt require ssl = False 2021-11-11T20:56:14.095 INFO:teuthology.orchestra.run.smithi096.stdout: rgw crypt s3 kms backend = testing 2021-11-11T20:56:14.096 INFO:teuthology.orchestra.run.smithi096.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2021-11-11T20:56:14.097 INFO:teuthology.orchestra.run.smithi096.stdout: rgw curl low speed time = 300 2021-11-11T20:56:14.097 INFO:teuthology.orchestra.run.smithi096.stdout: rgw data log num shards = 4 2021-11-11T20:56:14.098 INFO:teuthology.orchestra.run.smithi096.stdout: rgw md log max shards = 4 2021-11-11T20:56:14.098 INFO:teuthology.orchestra.run.smithi096.stdout: rgw sync log trim interval = 0 2021-11-11T20:56:14.098 INFO:teuthology.orchestra.run.smithi096.stdout: rgw sync meta inject err probability = 0.1 2021-11-11T20:56:14.098 INFO:teuthology.orchestra.run.smithi096.stdout: rgw sync obj etag verify = True 2021-11-11T20:56:14.098 INFO:teuthology.orchestra.run.smithi096.stdout: setgroup = ceph 2021-11-11T20:56:14.099 INFO:teuthology.orchestra.run.smithi096.stdout: setuser = ceph 2021-11-11T20:56:14.099 INFO:teuthology.orchestra.run.smithi096.stdout:[mon.a] 2021-11-11T20:56:14.099 INFO:teuthology.orchestra.run.smithi096.stdout:[osd.0] 2021-11-11T20:56:14.099 INFO:teuthology.orchestra.run.smithi096.stdout: osd_max_omap_entries_per_request = 10 2021-11-11T20:56:14.100 INFO:teuthology.orchestra.run.smithi096.stdout:[osd.1] 2021-11-11T20:56:14.100 INFO:teuthology.orchestra.run.smithi096.stdout: osd_max_omap_entries_per_request = 1000 2021-11-11T20:56:14.100 INFO:teuthology.orchestra.run.smithi096.stdout:[osd.2] 2021-11-11T20:56:14.100 INFO:teuthology.orchestra.run.smithi096.stdout: osd_max_omap_entries_per_request = 10000 2021-11-11T20:56:14.102 INFO:teuthology.orchestra.run.smithi198.stdout:[global] 2021-11-11T20:56:14.103 INFO:teuthology.orchestra.run.smithi198.stdout: chdir = "" 2021-11-11T20:56:14.103 INFO:teuthology.orchestra.run.smithi198.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2021-11-11T20:56:14.103 INFO:teuthology.orchestra.run.smithi198.stdout: auth supported = cephx 2021-11-11T20:56:14.103 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.104 INFO:teuthology.orchestra.run.smithi198.stdout: filestore xattr use omap = true 2021-11-11T20:56:14.104 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.104 INFO:teuthology.orchestra.run.smithi198.stdout: mon clock drift allowed = 1.000 2021-11-11T20:56:14.104 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.105 INFO:teuthology.orchestra.run.smithi198.stdout: osd crush chooseleaf type = 0 2021-11-11T20:56:14.105 INFO:teuthology.orchestra.run.smithi198.stdout: auth debug = true 2021-11-11T20:56:14.105 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.105 INFO:teuthology.orchestra.run.smithi198.stdout: ms die on old message = true 2021-11-11T20:56:14.106 INFO:teuthology.orchestra.run.smithi198.stdout: ms die on bug = true 2021-11-11T20:56:14.106 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.106 INFO:teuthology.orchestra.run.smithi198.stdout: mon max pg per osd = 10000 # >= luminous 2021-11-11T20:56:14.106 INFO:teuthology.orchestra.run.smithi198.stdout: mon pg warn max object skew = 0 2021-11-11T20:56:14.107 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.107 INFO:teuthology.orchestra.run.smithi198.stdout: # disable pg_autoscaler by default for new pools 2021-11-11T20:56:14.107 INFO:teuthology.orchestra.run.smithi198.stdout: osd_pool_default_pg_autoscale_mode = off 2021-11-11T20:56:14.107 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.108 INFO:teuthology.orchestra.run.smithi198.stdout: osd pool default size = 2 2021-11-11T20:56:14.108 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.108 INFO:teuthology.orchestra.run.smithi198.stdout: mon osd allow primary affinity = true 2021-11-11T20:56:14.108 INFO:teuthology.orchestra.run.smithi198.stdout: mon osd allow pg remap = true 2021-11-11T20:56:14.108 INFO:teuthology.orchestra.run.smithi198.stdout: mon warn on legacy crush tunables = false 2021-11-11T20:56:14.109 INFO:teuthology.orchestra.run.smithi198.stdout: mon warn on crush straw calc version zero = false 2021-11-11T20:56:14.109 INFO:teuthology.orchestra.run.smithi198.stdout: mon warn on no sortbitwise = false 2021-11-11T20:56:14.109 INFO:teuthology.orchestra.run.smithi198.stdout: mon warn on osd down out interval zero = false 2021-11-11T20:56:14.109 INFO:teuthology.orchestra.run.smithi198.stdout: mon warn on too few osds = false 2021-11-11T20:56:14.110 INFO:teuthology.orchestra.run.smithi198.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2021-11-11T20:56:14.110 INFO:teuthology.orchestra.run.smithi198.stdout: mon_warn_on_pool_no_redundancy = false 2021-11-11T20:56:14.110 INFO:teuthology.orchestra.run.smithi198.stdout: mon_allow_pool_size_one = true 2021-11-11T20:56:14.110 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.110 INFO:teuthology.orchestra.run.smithi198.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2021-11-11T20:56:14.111 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.111 INFO:teuthology.orchestra.run.smithi198.stdout: osd default data pool replay window = 5 2021-11-11T20:56:14.111 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.111 INFO:teuthology.orchestra.run.smithi198.stdout: mon allow pool delete = true 2021-11-11T20:56:14.111 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.112 INFO:teuthology.orchestra.run.smithi198.stdout: mon cluster log file level = debug 2021-11-11T20:56:14.112 INFO:teuthology.orchestra.run.smithi198.stdout: debug asserts on shutdown = true 2021-11-11T20:56:14.112 INFO:teuthology.orchestra.run.smithi198.stdout: mon health detail to clog = false 2021-11-11T20:56:14.112 INFO:teuthology.orchestra.run.smithi198.stdout: mon host = 172.21.15.96 2021-11-11T20:56:14.113 INFO:teuthology.orchestra.run.smithi198.stdout: fsid = 472dfe13-0d20-45f4-8fbc-1a0e45d2ce49 2021-11-11T20:56:14.113 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.113 INFO:teuthology.orchestra.run.smithi198.stdout:[osd] 2021-11-11T20:56:14.113 INFO:teuthology.orchestra.run.smithi198.stdout: osd journal size = 100 2021-11-11T20:56:14.113 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.114 INFO:teuthology.orchestra.run.smithi198.stdout: osd scrub load threshold = 5.0 2021-11-11T20:56:14.114 INFO:teuthology.orchestra.run.smithi198.stdout: osd scrub max interval = 600 2021-11-11T20:56:14.114 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.114 INFO:teuthology.orchestra.run.smithi198.stdout: osd recover clone overlap = true 2021-11-11T20:56:14.115 INFO:teuthology.orchestra.run.smithi198.stdout: osd recovery max chunk = 1048576 2021-11-11T20:56:14.115 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.115 INFO:teuthology.orchestra.run.smithi198.stdout: osd debug shutdown = true 2021-11-11T20:56:14.115 INFO:teuthology.orchestra.run.smithi198.stdout: osd debug op order = true 2021-11-11T20:56:14.115 INFO:teuthology.orchestra.run.smithi198.stdout: osd debug verify stray on activate = true 2021-11-11T20:56:14.116 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.116 INFO:teuthology.orchestra.run.smithi198.stdout: osd open classes on start = true 2021-11-11T20:56:14.116 INFO:teuthology.orchestra.run.smithi198.stdout: osd debug pg log writeout = true 2021-11-11T20:56:14.116 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.116 INFO:teuthology.orchestra.run.smithi198.stdout: osd deep scrub update digest min age = 30 2021-11-11T20:56:14.117 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.117 INFO:teuthology.orchestra.run.smithi198.stdout: osd map max advance = 10 2021-11-11T20:56:14.117 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.117 INFO:teuthology.orchestra.run.smithi198.stdout: journal zero on create = true 2021-11-11T20:56:14.118 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.118 INFO:teuthology.orchestra.run.smithi198.stdout: filestore ondisk finisher threads = 3 2021-11-11T20:56:14.118 INFO:teuthology.orchestra.run.smithi198.stdout: filestore apply finisher threads = 3 2021-11-11T20:56:14.118 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.119 INFO:teuthology.orchestra.run.smithi198.stdout: bdev debug aio = true 2021-11-11T20:56:14.119 INFO:teuthology.orchestra.run.smithi198.stdout: osd debug misdirected ops = true 2021-11-11T20:56:14.119 INFO:teuthology.orchestra.run.smithi198.stdout: debug filestore = 20 2021-11-11T20:56:14.119 INFO:teuthology.orchestra.run.smithi198.stdout: debug journal = 20 2021-11-11T20:56:14.119 INFO:teuthology.orchestra.run.smithi198.stdout: debug ms = 20 2021-11-11T20:56:14.120 INFO:teuthology.orchestra.run.smithi198.stdout: debug osd = 25 2021-11-11T20:56:14.120 INFO:teuthology.orchestra.run.smithi198.stdout: rocksdb delete range threshold = 0 2021-11-11T20:56:14.120 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.120 INFO:teuthology.orchestra.run.smithi198.stdout:[mgr] 2021-11-11T20:56:14.120 INFO:teuthology.orchestra.run.smithi198.stdout: debug ms = 1 2021-11-11T20:56:14.121 INFO:teuthology.orchestra.run.smithi198.stdout: debug mgr = 20 2021-11-11T20:56:14.121 INFO:teuthology.orchestra.run.smithi198.stdout: debug mon = 20 2021-11-11T20:56:14.121 INFO:teuthology.orchestra.run.smithi198.stdout: debug auth = 20 2021-11-11T20:56:14.121 INFO:teuthology.orchestra.run.smithi198.stdout: mon reweight min pgs per osd = 4 2021-11-11T20:56:14.122 INFO:teuthology.orchestra.run.smithi198.stdout: mon reweight min bytes per osd = 10 2021-11-11T20:56:14.122 INFO:teuthology.orchestra.run.smithi198.stdout: mgr/telemetry/nag = false 2021-11-11T20:56:14.122 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.122 INFO:teuthology.orchestra.run.smithi198.stdout:[mon] 2021-11-11T20:56:14.123 INFO:teuthology.orchestra.run.smithi198.stdout: debug ms = 1 2021-11-11T20:56:14.123 INFO:teuthology.orchestra.run.smithi198.stdout: debug mon = 20 2021-11-11T20:56:14.123 INFO:teuthology.orchestra.run.smithi198.stdout: debug paxos = 20 2021-11-11T20:56:14.123 INFO:teuthology.orchestra.run.smithi198.stdout: debug auth = 20 2021-11-11T20:56:14.123 INFO:teuthology.orchestra.run.smithi198.stdout: mon data avail warn = 5 2021-11-11T20:56:14.124 INFO:teuthology.orchestra.run.smithi198.stdout: mon mgr mkfs grace = 240 2021-11-11T20:56:14.124 INFO:teuthology.orchestra.run.smithi198.stdout: mon reweight min pgs per osd = 4 2021-11-11T20:56:14.124 INFO:teuthology.orchestra.run.smithi198.stdout: mon osd reporter subtree level = osd 2021-11-11T20:56:14.124 INFO:teuthology.orchestra.run.smithi198.stdout: mon osd prime pg temp = true 2021-11-11T20:56:14.125 INFO:teuthology.orchestra.run.smithi198.stdout: mon reweight min bytes per osd = 10 2021-11-11T20:56:14.125 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.125 INFO:teuthology.orchestra.run.smithi198.stdout: # rotate auth tickets quickly to exercise renewal paths 2021-11-11T20:56:14.125 INFO:teuthology.orchestra.run.smithi198.stdout: auth mon ticket ttl = 660 # 11m 2021-11-11T20:56:14.126 INFO:teuthology.orchestra.run.smithi198.stdout: auth service ticket ttl = 240 # 4m 2021-11-11T20:56:14.126 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.126 INFO:teuthology.orchestra.run.smithi198.stdout: # don't complain about insecure global_id in the test suite 2021-11-11T20:56:14.126 INFO:teuthology.orchestra.run.smithi198.stdout: mon_warn_on_insecure_global_id_reclaim = false 2021-11-11T20:56:14.126 INFO:teuthology.orchestra.run.smithi198.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2021-11-11T20:56:14.127 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:14.127 INFO:teuthology.orchestra.run.smithi198.stdout:[client] 2021-11-11T20:56:14.127 INFO:teuthology.orchestra.run.smithi198.stdout: rgw cache enabled = true 2021-11-11T20:56:14.127 INFO:teuthology.orchestra.run.smithi198.stdout: rgw enable ops log = true 2021-11-11T20:56:14.128 INFO:teuthology.orchestra.run.smithi198.stdout: rgw enable usage log = true 2021-11-11T20:56:14.128 INFO:teuthology.orchestra.run.smithi198.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2021-11-11T20:56:14.128 INFO:teuthology.orchestra.run.smithi198.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2021-11-11T20:56:14.128 INFO:teuthology.orchestra.run.smithi198.stdout: debug rgw = 20 2021-11-11T20:56:14.128 INFO:teuthology.orchestra.run.smithi198.stdout: rgw crypt require ssl = False 2021-11-11T20:56:14.129 INFO:teuthology.orchestra.run.smithi198.stdout: rgw crypt s3 kms backend = testing 2021-11-11T20:56:14.129 INFO:teuthology.orchestra.run.smithi198.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2021-11-11T20:56:14.129 INFO:teuthology.orchestra.run.smithi198.stdout: rgw curl low speed time = 300 2021-11-11T20:56:14.129 INFO:teuthology.orchestra.run.smithi198.stdout: rgw data log num shards = 4 2021-11-11T20:56:14.130 INFO:teuthology.orchestra.run.smithi198.stdout: rgw md log max shards = 4 2021-11-11T20:56:14.130 INFO:teuthology.orchestra.run.smithi198.stdout: rgw sync log trim interval = 0 2021-11-11T20:56:14.130 INFO:teuthology.orchestra.run.smithi198.stdout: rgw sync meta inject err probability = 0.1 2021-11-11T20:56:14.130 INFO:teuthology.orchestra.run.smithi198.stdout: rgw sync obj etag verify = True 2021-11-11T20:56:14.131 INFO:teuthology.orchestra.run.smithi198.stdout: setgroup = ceph 2021-11-11T20:56:14.131 INFO:teuthology.orchestra.run.smithi198.stdout: setuser = ceph 2021-11-11T20:56:14.131 INFO:teuthology.orchestra.run.smithi198.stdout:[mon.a] 2021-11-11T20:56:14.131 INFO:teuthology.orchestra.run.smithi198.stdout:[osd.0] 2021-11-11T20:56:14.131 INFO:teuthology.orchestra.run.smithi198.stdout: osd_max_omap_entries_per_request = 10 2021-11-11T20:56:14.132 INFO:teuthology.orchestra.run.smithi198.stdout:[osd.1] 2021-11-11T20:56:14.132 INFO:teuthology.orchestra.run.smithi198.stdout: osd_max_omap_entries_per_request = 1000 2021-11-11T20:56:14.132 INFO:teuthology.orchestra.run.smithi198.stdout:[osd.2] 2021-11-11T20:56:14.132 INFO:teuthology.orchestra.run.smithi198.stdout: osd_max_omap_entries_per_request = 10000 2021-11-11T20:56:14.133 INFO:tasks.ceph:Creating admin key on c1.mon.a... 2021-11-11T20:56:14.133 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/c1.keyring 2021-11-11T20:56:14.191 INFO:tasks.ceph:Copying monmap to all nodes... 2021-11-11T20:56:14.191 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:14.192 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/etc/ceph/c1.keyring of=/dev/stdout 2021-11-11T20:56:14.206 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:14.206 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/home/ubuntu/cephtest/c1.monmap of=/dev/stdout 2021-11-11T20:56:14.260 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi096.front.sepia.ceph.com 2021-11-11T20:56:14.260 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:14.261 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/ceph/c1.keyring 2021-11-11T20:56:14.261 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod 0644 /etc/ceph/c1.keyring 2021-11-11T20:56:14.332 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:14.333 DEBUG:teuthology.orchestra.run.smithi096:> dd of=/home/ubuntu/cephtest/c1.monmap 2021-11-11T20:56:14.387 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi198.front.sepia.ceph.com 2021-11-11T20:56:14.387 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:14.387 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd of=/etc/ceph/c1.keyring 2021-11-11T20:56:14.388 DEBUG:teuthology.orchestra.run.smithi198:> sudo chmod 0644 /etc/ceph/c1.keyring 2021-11-11T20:56:14.427 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:14.428 DEBUG:teuthology.orchestra.run.smithi198:> dd of=/home/ubuntu/cephtest/c1.monmap 2021-11-11T20:56:14.483 INFO:tasks.ceph:Setting up mon nodes... 2021-11-11T20:56:14.484 INFO:tasks.ceph:Setting up mgr nodes... 2021-11-11T20:56:14.484 DEBUG:teuthology.orchestra.run.smithi096:> sudo mkdir -p /var/lib/ceph/mgr/c1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/c1-x/keyring 2021-11-11T20:56:14.530 INFO:teuthology.orchestra.run.smithi096.stdout:creating /var/lib/ceph/mgr/c1-x/keyring 2021-11-11T20:56:14.532 INFO:tasks.ceph:Setting up mds nodes... 2021-11-11T20:56:14.533 INFO:tasks.ceph_client:Setting up client nodes... 2021-11-11T20:56:14.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/c1.client.0.keyring && sudo chmod 0644 /etc/ceph/c1.client.0.keyring 2021-11-11T20:56:14.569 INFO:teuthology.orchestra.run.smithi096.stdout:creating /etc/ceph/c1.client.0.keyring 2021-11-11T20:56:14.582 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/c1.client.1.keyring && sudo chmod 0644 /etc/ceph/c1.client.1.keyring 2021-11-11T20:56:14.658 INFO:teuthology.orchestra.run.smithi096.stdout:creating /etc/ceph/c1.client.1.keyring 2021-11-11T20:56:14.670 INFO:tasks.ceph:Running mkfs on osd nodes... 2021-11-11T20:56:14.670 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi096.front.sepia.ceph.com'): {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'}} 2021-11-11T20:56:14.671 DEBUG:teuthology.orchestra.run.smithi096:> sudo mkdir -p /var/lib/ceph/osd/c1-0 2021-11-11T20:56:14.734 INFO:tasks.ceph:roles_to_devs: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2021-11-11T20:56:14.734 INFO:tasks.ceph:role: c1.osd.0 2021-11-11T20:56:14.734 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi096.front.sepia.ceph.com 2021-11-11T20:56:14.735 DEBUG:teuthology.orchestra.run.smithi096:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2021-11-11T20:56:14.805 INFO:teuthology.orchestra.run.smithi096.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2021-11-11T20:56:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: = sectsz=512 attr=2, projid32bit=1 2021-11-11T20:56:14.805 INFO:teuthology.orchestra.run.smithi096.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2021-11-11T20:56:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: = reflink=1 2021-11-11T20:56:14.806 INFO:teuthology.orchestra.run.smithi096.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2021-11-11T20:56:14.806 INFO:teuthology.orchestra.run.smithi096.stdout: = sunit=0 swidth=0 blks 2021-11-11T20:56:14.806 INFO:teuthology.orchestra.run.smithi096.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2021-11-11T20:56:14.807 INFO:teuthology.orchestra.run.smithi096.stdout:log =internal log bsize=4096 blocks=11443, version=2 2021-11-11T20:56:14.807 INFO:teuthology.orchestra.run.smithi096.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2021-11-11T20:56:14.807 INFO:teuthology.orchestra.run.smithi096.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2021-11-11T20:56:15.176 INFO:teuthology.orchestra.run.smithi096.stdout:Discarding blocks...Done. 2021-11-11T20:56:15.177 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi096.front.sepia.ceph.com -o noatime 2021-11-11T20:56:15.178 DEBUG:teuthology.orchestra.run.smithi096:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/c1-0 2021-11-11T20:56:15.304 DEBUG:teuthology.orchestra.run.smithi096:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-0 2021-11-11T20:56:15.331 DEBUG:teuthology.orchestra.run.smithi096:> sudo mkdir -p /var/lib/ceph/osd/c1-1 2021-11-11T20:56:15.395 INFO:tasks.ceph:roles_to_devs: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2021-11-11T20:56:15.395 INFO:tasks.ceph:role: c1.osd.1 2021-11-11T20:56:15.395 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi096.front.sepia.ceph.com 2021-11-11T20:56:15.396 DEBUG:teuthology.orchestra.run.smithi096:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2021-11-11T20:56:15.466 INFO:teuthology.orchestra.run.smithi096.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2021-11-11T20:56:15.466 INFO:teuthology.orchestra.run.smithi096.stdout: = sectsz=512 attr=2, projid32bit=1 2021-11-11T20:56:15.467 INFO:teuthology.orchestra.run.smithi096.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2021-11-11T20:56:15.467 INFO:teuthology.orchestra.run.smithi096.stdout: = reflink=1 2021-11-11T20:56:15.467 INFO:teuthology.orchestra.run.smithi096.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2021-11-11T20:56:15.467 INFO:teuthology.orchestra.run.smithi096.stdout: = sunit=0 swidth=0 blks 2021-11-11T20:56:15.468 INFO:teuthology.orchestra.run.smithi096.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2021-11-11T20:56:15.468 INFO:teuthology.orchestra.run.smithi096.stdout:log =internal log bsize=4096 blocks=11443, version=2 2021-11-11T20:56:15.468 INFO:teuthology.orchestra.run.smithi096.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2021-11-11T20:56:15.468 INFO:teuthology.orchestra.run.smithi096.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2021-11-11T20:56:15.828 INFO:teuthology.orchestra.run.smithi096.stdout:Discarding blocks...Done. 2021-11-11T20:56:15.829 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi096.front.sepia.ceph.com -o noatime 2021-11-11T20:56:15.830 DEBUG:teuthology.orchestra.run.smithi096:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/c1-1 2021-11-11T20:56:15.912 DEBUG:teuthology.orchestra.run.smithi096:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-1 2021-11-11T20:56:15.980 DEBUG:teuthology.orchestra.run.smithi096:> sudo mkdir -p /var/lib/ceph/osd/c1-2 2021-11-11T20:56:16.043 INFO:tasks.ceph:roles_to_devs: {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'} 2021-11-11T20:56:16.044 INFO:tasks.ceph:role: c1.osd.2 2021-11-11T20:56:16.044 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi096.front.sepia.ceph.com 2021-11-11T20:56:16.045 DEBUG:teuthology.orchestra.run.smithi096:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2021-11-11T20:56:16.115 INFO:teuthology.orchestra.run.smithi096.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2021-11-11T20:56:16.115 INFO:teuthology.orchestra.run.smithi096.stdout: = sectsz=512 attr=2, projid32bit=1 2021-11-11T20:56:16.116 INFO:teuthology.orchestra.run.smithi096.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2021-11-11T20:56:16.116 INFO:teuthology.orchestra.run.smithi096.stdout: = reflink=1 2021-11-11T20:56:16.116 INFO:teuthology.orchestra.run.smithi096.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2021-11-11T20:56:16.116 INFO:teuthology.orchestra.run.smithi096.stdout: = sunit=0 swidth=0 blks 2021-11-11T20:56:16.117 INFO:teuthology.orchestra.run.smithi096.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2021-11-11T20:56:16.117 INFO:teuthology.orchestra.run.smithi096.stdout:log =internal log bsize=4096 blocks=11443, version=2 2021-11-11T20:56:16.117 INFO:teuthology.orchestra.run.smithi096.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2021-11-11T20:56:16.117 INFO:teuthology.orchestra.run.smithi096.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2021-11-11T20:56:16.490 INFO:teuthology.orchestra.run.smithi096.stdout:Discarding blocks...Done. 2021-11-11T20:56:16.491 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi096.front.sepia.ceph.com -o noatime 2021-11-11T20:56:16.492 DEBUG:teuthology.orchestra.run.smithi096:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/c1-2 2021-11-11T20:56:16.594 DEBUG:teuthology.orchestra.run.smithi096:> sudo /sbin/restorecon /var/lib/ceph/osd/c1-2 2021-11-11T20:56:16.620 DEBUG:teuthology.orchestra.run.smithi096:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/c1.monmap 2021-11-11T20:56:16.701 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:16.700+0000 7fb9054db0c0 -1 auth: error reading file: /var/lib/ceph/osd/c1-0/keyring: can't open /var/lib/ceph/osd/c1-0/keyring: (2) No such file or directory 2021-11-11T20:56:16.701 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:16.701+0000 7fb9054db0c0 -1 created new key in keyring /var/lib/ceph/osd/c1-0/keyring 2021-11-11T20:56:16.702 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:16.701+0000 7fb9054db0c0 -1 bluestore(/var/lib/ceph/osd/c1-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-0/block: (2) No such file or directory 2021-11-11T20:56:16.702 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:16.701+0000 7fb9054db0c0 -1 bluestore(/var/lib/ceph/osd/c1-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-0/block: (2) No such file or directory 2021-11-11T20:56:16.702 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:16.701+0000 7fb9054db0c0 -1 bluestore(/var/lib/ceph/osd/c1-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-0/block: (2) No such file or directory 2021-11-11T20:56:16.708 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:16.709+0000 7fb9054db0c0 -1 bluestore(/var/lib/ceph/osd/c1-0) _read_fsid unparsable uuid 2021-11-11T20:56:18.789 DEBUG:teuthology.orchestra.run.smithi096:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-0 2021-11-11T20:56:18.858 DEBUG:teuthology.orchestra.run.smithi096:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/c1.monmap 2021-11-11T20:56:18.941 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:18.941+0000 7f5720c010c0 -1 auth: error reading file: /var/lib/ceph/osd/c1-1/keyring: can't open /var/lib/ceph/osd/c1-1/keyring: (2) No such file or directory 2021-11-11T20:56:18.942 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:18.941+0000 7f5720c010c0 -1 created new key in keyring /var/lib/ceph/osd/c1-1/keyring 2021-11-11T20:56:18.942 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:18.941+0000 7f5720c010c0 -1 bluestore(/var/lib/ceph/osd/c1-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-1/block: (2) No such file or directory 2021-11-11T20:56:18.943 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:18.941+0000 7f5720c010c0 -1 bluestore(/var/lib/ceph/osd/c1-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-1/block: (2) No such file or directory 2021-11-11T20:56:18.943 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:18.941+0000 7f5720c010c0 -1 bluestore(/var/lib/ceph/osd/c1-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-1/block: (2) No such file or directory 2021-11-11T20:56:18.944 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:18.942+0000 7f5720c010c0 -1 bluestore(/var/lib/ceph/osd/c1-1) _read_fsid unparsable uuid 2021-11-11T20:56:21.021 DEBUG:teuthology.orchestra.run.smithi096:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-1 2021-11-11T20:56:21.089 DEBUG:teuthology.orchestra.run.smithi096:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/c1.monmap 2021-11-11T20:56:21.173 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:21.173+0000 7f7c87ec10c0 -1 auth: error reading file: /var/lib/ceph/osd/c1-2/keyring: can't open /var/lib/ceph/osd/c1-2/keyring: (2) No such file or directory 2021-11-11T20:56:21.173 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:21.173+0000 7f7c87ec10c0 -1 created new key in keyring /var/lib/ceph/osd/c1-2/keyring 2021-11-11T20:56:21.173 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:21.173+0000 7f7c87ec10c0 -1 bluestore(/var/lib/ceph/osd/c1-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-2/block: (2) No such file or directory 2021-11-11T20:56:21.173 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:21.173+0000 7f7c87ec10c0 -1 bluestore(/var/lib/ceph/osd/c1-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-2/block: (2) No such file or directory 2021-11-11T20:56:21.174 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:21.173+0000 7f7c87ec10c0 -1 bluestore(/var/lib/ceph/osd/c1-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c1-2/block: (2) No such file or directory 2021-11-11T20:56:21.174 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:56:21.173+0000 7f7c87ec10c0 -1 bluestore(/var/lib/ceph/osd/c1-2) _read_fsid unparsable uuid 2021-11-11T20:56:23.256 DEBUG:teuthology.orchestra.run.smithi096:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c1-2 2021-11-11T20:56:23.326 INFO:tasks.ceph:Reading keys from all nodes... 2021-11-11T20:56:23.327 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:23.327 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/var/lib/ceph/mgr/c1-x/keyring of=/dev/stdout 2021-11-11T20:56:23.393 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:23.394 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/var/lib/ceph/osd/c1-0/keyring of=/dev/stdout 2021-11-11T20:56:23.459 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:23.460 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/var/lib/ceph/osd/c1-1/keyring of=/dev/stdout 2021-11-11T20:56:23.524 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:23.525 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/var/lib/ceph/osd/c1-2/keyring of=/dev/stdout 2021-11-11T20:56:23.589 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:23.589 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/etc/ceph/c1.client.0.keyring of=/dev/stdout 2021-11-11T20:56:23.644 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:23.644 DEBUG:teuthology.orchestra.run.smithi096:> dd if=/etc/ceph/c1.client.1.keyring of=/dev/stdout 2021-11-11T20:56:23.699 INFO:tasks.ceph:Adding keys to all mons... 2021-11-11T20:56:23.699 DEBUG:teuthology.orchestra.run.smithi096:> sudo tee -a /etc/ceph/c1.keyring 2021-11-11T20:56:23.762 INFO:teuthology.orchestra.run.smithi096.stdout:[mgr.x] 2021-11-11T20:56:23.763 INFO:teuthology.orchestra.run.smithi096.stdout: key = AQBug41hKcq2HxAAHxlAMTRJRPpbMdgzkX3kKA== 2021-11-11T20:56:23.764 INFO:teuthology.orchestra.run.smithi096.stdout:[osd.0] 2021-11-11T20:56:23.764 INFO:teuthology.orchestra.run.smithi096.stdout: key = AQBwg41h0ajfKRAA32n+F3hi7xjTq0x2ZsK7Cg== 2021-11-11T20:56:23.764 INFO:teuthology.orchestra.run.smithi096.stdout:[osd.1] 2021-11-11T20:56:23.765 INFO:teuthology.orchestra.run.smithi096.stdout: key = AQByg41hZeUoOBAAyZnuznmdibffFFc2qwhH2Q== 2021-11-11T20:56:23.765 INFO:teuthology.orchestra.run.smithi096.stdout:[osd.2] 2021-11-11T20:56:23.765 INFO:teuthology.orchestra.run.smithi096.stdout: key = AQB1g41hA+leChAAE7sPBB4Rp4bUMLbwSD5R6g== 2021-11-11T20:56:23.765 INFO:teuthology.orchestra.run.smithi096.stdout:[client.0] 2021-11-11T20:56:23.765 INFO:teuthology.orchestra.run.smithi096.stdout: key = AQBug41h/GIQIhAAt+g2SKIE+q4v4I1yuhm7Bw== 2021-11-11T20:56:23.766 INFO:teuthology.orchestra.run.smithi096.stdout:[client.1] 2021-11-11T20:56:23.766 INFO:teuthology.orchestra.run.smithi096.stdout: key = AQBug41hEclWJxAA7863BHW8e+NKpxF8HVTgNg== 2021-11-11T20:56:23.767 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2021-11-11T20:56:23.870 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2021-11-11T20:56:23.922 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2021-11-11T20:56:23.976 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2021-11-11T20:56:24.038 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2021-11-11T20:56:24.098 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c1.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2021-11-11T20:56:24.155 INFO:tasks.ceph:Running mkfs on mon nodes... 2021-11-11T20:56:24.156 DEBUG:teuthology.orchestra.run.smithi096:> sudo mkdir -p /var/lib/ceph/mon/c1-a 2021-11-11T20:56:24.184 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster c1 --mkfs -i a --monmap /home/ubuntu/cephtest/c1.monmap --keyring /etc/ceph/c1.keyring 2021-11-11T20:56:24.276 DEBUG:teuthology.orchestra.run.smithi096:> sudo chown -R ceph:ceph /var/lib/ceph/mon/c1-a 2021-11-11T20:56:24.300 DEBUG:teuthology.orchestra.run.smithi096:> rm -- /home/ubuntu/cephtest/c1.monmap 2021-11-11T20:56:24.354 INFO:tasks.ceph:Starting mon daemons in cluster c1... 2021-11-11T20:56:24.355 INFO:tasks.ceph.c1.mon.a:Restarting daemon 2021-11-11T20:56:24.355 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster c1 -i a 2021-11-11T20:56:24.396 INFO:tasks.ceph.c1.mon.a:Started 2021-11-11T20:56:24.397 INFO:tasks.ceph:Starting mgr daemons in cluster c1... 2021-11-11T20:56:24.397 INFO:tasks.ceph.c1.mgr.x:Restarting daemon 2021-11-11T20:56:24.397 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster c1 -i x 2021-11-11T20:56:24.400 INFO:tasks.ceph.c1.mgr.x:Started 2021-11-11T20:56:24.400 INFO:tasks.ceph:Setting crush tunables to default 2021-11-11T20:56:24.400 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph --cluster c1 osd crush tunables default 2021-11-11T20:56:24.610 INFO:teuthology.orchestra.run.smithi096.stderr:adjusted tunables profile to default 2021-11-11T20:56:24.621 INFO:tasks.ceph:Starting osd daemons in cluster c1... 2021-11-11T20:56:24.622 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:24.622 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/var/lib/ceph/osd/c1-0/fsid of=/dev/stdout 2021-11-11T20:56:24.646 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:24.646 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/var/lib/ceph/osd/c1-1/fsid of=/dev/stdout 2021-11-11T20:56:24.710 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:24.710 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd if=/var/lib/ceph/osd/c1-2/fsid of=/dev/stdout 2021-11-11T20:56:24.775 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph --cluster c1 osd new e660e671-21bc-4df9-9ec6-f724fd1694dd 0 2021-11-11T20:56:25.024 INFO:teuthology.orchestra.run.smithi096.stdout:0 2021-11-11T20:56:25.036 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph --cluster c1 osd new 27ed9b63-62ad-4e59-9c9e-cb1eb616a5b5 1 2021-11-11T20:56:25.256 INFO:teuthology.orchestra.run.smithi096.stdout:1 2021-11-11T20:56:25.270 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph --cluster c1 osd new a173321f-4479-41a3-a3d1-33558b8ec04d 2 2021-11-11T20:56:25.484 INFO:teuthology.orchestra.run.smithi096.stdout:2 2021-11-11T20:56:25.497 INFO:tasks.ceph.c1.osd.0:Restarting daemon 2021-11-11T20:56:25.497 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 0 2021-11-11T20:56:25.499 INFO:tasks.ceph.c1.osd.0:Started 2021-11-11T20:56:25.500 INFO:tasks.ceph.c1.osd.1:Restarting daemon 2021-11-11T20:56:25.500 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 1 2021-11-11T20:56:25.503 INFO:tasks.ceph.c1.osd.1:Started 2021-11-11T20:56:25.503 INFO:tasks.ceph.c1.osd.2:Restarting daemon 2021-11-11T20:56:25.503 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c1 -i 2 2021-11-11T20:56:25.506 INFO:tasks.ceph.c1.osd.2:Started 2021-11-11T20:56:25.506 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd dump --format=json 2021-11-11T20:56:25.722 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:25.723 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":5,"fsid":"472dfe13-0d20-45f4-8fbc-1a0e45d2ce49","created":"2021-11-11T20:56:24.512976+0000","modified":"2021-11-11T20:56:25.479405+0000","last_up_change":"0.000000","last_in_change":"2021-11-11T20:56:25.479405+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[],"osds":[{"osd":0,"uuid":"e660e671-21bc-4df9-9ec6-f724fd1694dd","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"27ed9b63-62ad-4e59-9c9e-cb1eb616a5b5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"a173321f-4479-41a3-a3d1-33558b8ec04d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2021-11-11T20:56:25.735 INFO:tasks.ceph.ceph_manager.c1:[] 2021-11-11T20:56:25.735 INFO:tasks.ceph:Waiting for OSDs to come up 2021-11-11T20:56:25.865 INFO:tasks.ceph.c1.osd.0.smithi096.stderr:2021-11-11T20:56:25.866+0000 7fd59e5940c0 -1 Falling back to public interface 2021-11-11T20:56:26.038 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c1 osd dump --format=json 2021-11-11T20:56:26.114 INFO:tasks.ceph.c1.osd.1.smithi096.stderr:2021-11-11T20:56:26.115+0000 7fed7869d0c0 -1 Falling back to public interface 2021-11-11T20:56:26.121 INFO:tasks.ceph.c1.osd.2.smithi096.stderr:2021-11-11T20:56:26.122+0000 7f798dec40c0 -1 Falling back to public interface 2021-11-11T20:56:26.222 INFO:teuthology.misc.health.smithi096.stdout: 2021-11-11T20:56:26.222 INFO:teuthology.misc.health.smithi096.stdout:{"epoch":5,"fsid":"472dfe13-0d20-45f4-8fbc-1a0e45d2ce49","created":"2021-11-11T20:56:24.512976+0000","modified":"2021-11-11T20:56:25.479405+0000","last_up_change":"0.000000","last_in_change":"2021-11-11T20:56:25.479405+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[],"osds":[{"osd":0,"uuid":"e660e671-21bc-4df9-9ec6-f724fd1694dd","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"27ed9b63-62ad-4e59-9c9e-cb1eb616a5b5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"a173321f-4479-41a3-a3d1-33558b8ec04d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2021-11-11T20:56:26.233 DEBUG:teuthology.misc:0 of 3 OSDs are up 2021-11-11T20:56:28.637 INFO:tasks.ceph.c1.osd.0.smithi096.stderr:2021-11-11T20:56:28.637+0000 7fd59e5940c0 -1 osd.0 0 log_to_monitors {default=true} 2021-11-11T20:56:28.831 INFO:tasks.ceph.c1.osd.1.smithi096.stderr:2021-11-11T20:56:28.831+0000 7fed7869d0c0 -1 osd.1 0 log_to_monitors {default=true} 2021-11-11T20:56:28.848 INFO:tasks.ceph.c1.osd.2.smithi096.stderr:2021-11-11T20:56:28.848+0000 7f798dec40c0 -1 osd.2 0 log_to_monitors {default=true} 2021-11-11T20:56:30.315 INFO:tasks.ceph.c1.osd.0.smithi096.stderr:2021-11-11T20:56:30.316+0000 7fd594f52700 -1 osd.0 0 waiting for initial osdmap 2021-11-11T20:56:30.317 INFO:tasks.ceph.c1.osd.0.smithi096.stderr:2021-11-11T20:56:30.318+0000 7fd58d8e5700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2021-11-11T20:56:31.632 INFO:tasks.ceph.c1.osd.2.smithi096.stderr:2021-11-11T20:56:31.633+0000 7f7984882700 -1 osd.2 0 waiting for initial osdmap 2021-11-11T20:56:31.634 INFO:tasks.ceph.c1.osd.2.smithi096.stderr:2021-11-11T20:56:31.635+0000 7f797d215700 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2021-11-11T20:56:31.743 INFO:tasks.ceph.c1.osd.1.smithi096.stderr:2021-11-11T20:56:31.743+0000 7fed6f05b700 -1 osd.1 0 waiting for initial osdmap 2021-11-11T20:56:31.744 INFO:tasks.ceph.c1.osd.1.smithi096.stderr:2021-11-11T20:56:31.745+0000 7fed679ee700 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2021-11-11T20:56:32.536 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c1 osd dump --format=json 2021-11-11T20:56:32.822 INFO:teuthology.misc.health.smithi096.stdout: 2021-11-11T20:56:32.823 INFO:teuthology.misc.health.smithi096.stdout:{"epoch":9,"fsid":"472dfe13-0d20-45f4-8fbc-1a0e45d2ce49","created":"2021-11-11T20:56:24.512976+0000","modified":"2021-11-11T20:56:32.340838+0000","last_up_change":"2021-11-11T20:56:32.340838+0000","last_in_change":"2021-11-11T20:56:25.479405+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[],"osds":[{"osd":0,"uuid":"e660e671-21bc-4df9-9ec6-f724fd1694dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6800","nonce":32071},{"type":"v1","addr":"172.21.15.96:6801","nonce":32071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":32071},{"type":"v1","addr":"172.21.15.96:6803","nonce":32071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":32071},{"type":"v1","addr":"172.21.15.96:6807","nonce":32071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":32071},{"type":"v1","addr":"172.21.15.96:6805","nonce":32071}]},"public_addr":"172.21.15.96:6801/32071","cluster_addr":"172.21.15.96:6803/32071","heartbeat_back_addr":"172.21.15.96:6807/32071","heartbeat_front_addr":"172.21.15.96:6805/32071","state":["exists","up"]},{"osd":1,"uuid":"27ed9b63-62ad-4e59-9c9e-cb1eb616a5b5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":32072},{"type":"v1","addr":"172.21.15.96:6809","nonce":32072}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":32072},{"type":"v1","addr":"172.21.15.96:6811","nonce":32072}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":32072},{"type":"v1","addr":"172.21.15.96:6815","nonce":32072}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":32072},{"type":"v1","addr":"172.21.15.96:6813","nonce":32072}]},"public_addr":"172.21.15.96:6809/32072","cluster_addr":"172.21.15.96:6811/32072","heartbeat_back_addr":"172.21.15.96:6815/32072","heartbeat_front_addr":"172.21.15.96:6813/32072","state":["exists","up"]},{"osd":2,"uuid":"a173321f-4479-41a3-a3d1-33558b8ec04d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":32078},{"type":"v1","addr":"172.21.15.96:6817","nonce":32078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":32078},{"type":"v1","addr":"172.21.15.96:6819","nonce":32078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":32078},{"type":"v1","addr":"172.21.15.96:6823","nonce":32078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":32078},{"type":"v1","addr":"172.21.15.96:6821","nonce":32078}]},"public_addr":"172.21.15.96:6817/32078","cluster_addr":"172.21.15.96:6819/32078","heartbeat_back_addr":"172.21.15.96:6823/32078","heartbeat_front_addr":"172.21.15.96:6821/32078","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:29.669596+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2021-11-11T20:56:32.834 DEBUG:teuthology.misc:3 of 3 OSDs are up 2021-11-11T20:56:32.834 INFO:tasks.ceph:Creating RBD pool 2021-11-11T20:56:32.835 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph --cluster c1 osd pool create rbd 8 2021-11-11T20:56:33.644 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'rbd' created 2021-11-11T20:56:33.658 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph --cluster c1 osd pool application enable rbd rbd --yes-i-really-mean-it 2021-11-11T20:56:34.656 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rbd' on pool 'rbd' 2021-11-11T20:56:34.671 INFO:tasks.ceph:Starting mds daemons in cluster c1... 2021-11-11T20:56:34.671 INFO:tasks.ceph:Waiting until c1 daemons up and pgs clean... 2021-11-11T20:56:34.672 INFO:tasks.ceph.ceph_manager.c1:waiting for mgr available 2021-11-11T20:56:34.672 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 mgr dump --format=json 2021-11-11T20:56:34.673 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2021-11-11T20:56:35.003 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:35.014 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":4,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6824","nonce":31717},{"type":"v1","addr":"172.21.15.96:6825","nonce":31717}]},"active_addr":"172.21.15.96:6825/31717","active_change":"2021-11-11T20:56:29.645591+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Registry to which we should normalize unqualified image names","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"autoscale_profile":{"name":"autoscale_profile","type":"str","level":"advanced","flags":1,"default_value":"scale-up","min":"","max":"","enum_allowed":[],"desc":"pg_autoscale profiler","long_desc":"Determines the behavior of the autoscaler algorithm `scale-up` means that it starts out with minmum pgs and scales up when there is pressure, `scale-down` means starts out with full pgs and scales down when there is pressure ","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share perf counter metrics summed across the whole cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":0,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":2714203708}]},{"addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":51141647}]},{"addrvec":[{"type":"v2","addr":"172.21.15.96:0","nonce":443946026}]}]}} 2021-11-11T20:56:35.017 INFO:tasks.ceph.ceph_manager.c1:mgr available! 2021-11-11T20:56:35.017 INFO:tasks.ceph.ceph_manager.c1:waiting for all up 2021-11-11T20:56:35.017 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd dump --format=json 2021-11-11T20:56:35.297 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:35.297 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":11,"fsid":"472dfe13-0d20-45f4-8fbc-1a0e45d2ce49","created":"2021-11-11T20:56:24.512976+0000","modified":"2021-11-11T20:56:34.641957+0000","last_up_change":"2021-11-11T20:56:32.340838+0000","last_in_change":"2021-11-11T20:56:25.479405+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":"rbd","create_time":"2021-11-11T20:56:33.128653+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":2,"pool_name":".mgr","create_time":"2021-11-11T20:56:33.679221+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"e660e671-21bc-4df9-9ec6-f724fd1694dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6800","nonce":32071},{"type":"v1","addr":"172.21.15.96:6801","nonce":32071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":32071},{"type":"v1","addr":"172.21.15.96:6803","nonce":32071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":32071},{"type":"v1","addr":"172.21.15.96:6807","nonce":32071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":32071},{"type":"v1","addr":"172.21.15.96:6805","nonce":32071}]},"public_addr":"172.21.15.96:6801/32071","cluster_addr":"172.21.15.96:6803/32071","heartbeat_back_addr":"172.21.15.96:6807/32071","heartbeat_front_addr":"172.21.15.96:6805/32071","state":["exists","up"]},{"osd":1,"uuid":"27ed9b63-62ad-4e59-9c9e-cb1eb616a5b5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":32072},{"type":"v1","addr":"172.21.15.96:6809","nonce":32072}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":32072},{"type":"v1","addr":"172.21.15.96:6811","nonce":32072}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":32072},{"type":"v1","addr":"172.21.15.96:6815","nonce":32072}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":32072},{"type":"v1","addr":"172.21.15.96:6813","nonce":32072}]},"public_addr":"172.21.15.96:6809/32072","cluster_addr":"172.21.15.96:6811/32072","heartbeat_back_addr":"172.21.15.96:6815/32072","heartbeat_front_addr":"172.21.15.96:6813/32072","state":["exists","up"]},{"osd":2,"uuid":"a173321f-4479-41a3-a3d1-33558b8ec04d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":32078},{"type":"v1","addr":"172.21.15.96:6817","nonce":32078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":32078},{"type":"v1","addr":"172.21.15.96:6819","nonce":32078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":32078},{"type":"v1","addr":"172.21.15.96:6823","nonce":32078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":32078},{"type":"v1","addr":"172.21.15.96:6821","nonce":32078}]},"public_addr":"172.21.15.96:6817/32078","cluster_addr":"172.21.15.96:6819/32078","heartbeat_back_addr":"172.21.15.96:6823/32078","heartbeat_front_addr":"172.21.15.96:6821/32078","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:29.669596+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:29.800709+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:29.855947+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2021-11-11T20:56:35.310 INFO:tasks.ceph.ceph_manager.c1:all up! 2021-11-11T20:56:35.310 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd dump --format=json 2021-11-11T20:56:35.589 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:35.590 INFO:teuthology.orchestra.run.smithi096.stdout:{"epoch":11,"fsid":"472dfe13-0d20-45f4-8fbc-1a0e45d2ce49","created":"2021-11-11T20:56:24.512976+0000","modified":"2021-11-11T20:56:34.641957+0000","last_up_change":"2021-11-11T20:56:32.340838+0000","last_in_change":"2021-11-11T20:56:25.479405+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":"rbd","create_time":"2021-11-11T20:56:33.128653+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":2,"pool_name":".mgr","create_time":"2021-11-11T20:56:33.679221+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"e660e671-21bc-4df9-9ec6-f724fd1694dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6800","nonce":32071},{"type":"v1","addr":"172.21.15.96:6801","nonce":32071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6802","nonce":32071},{"type":"v1","addr":"172.21.15.96:6803","nonce":32071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6806","nonce":32071},{"type":"v1","addr":"172.21.15.96:6807","nonce":32071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6804","nonce":32071},{"type":"v1","addr":"172.21.15.96:6805","nonce":32071}]},"public_addr":"172.21.15.96:6801/32071","cluster_addr":"172.21.15.96:6803/32071","heartbeat_back_addr":"172.21.15.96:6807/32071","heartbeat_front_addr":"172.21.15.96:6805/32071","state":["exists","up"]},{"osd":1,"uuid":"27ed9b63-62ad-4e59-9c9e-cb1eb616a5b5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6808","nonce":32072},{"type":"v1","addr":"172.21.15.96:6809","nonce":32072}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6810","nonce":32072},{"type":"v1","addr":"172.21.15.96:6811","nonce":32072}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6814","nonce":32072},{"type":"v1","addr":"172.21.15.96:6815","nonce":32072}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6812","nonce":32072},{"type":"v1","addr":"172.21.15.96:6813","nonce":32072}]},"public_addr":"172.21.15.96:6809/32072","cluster_addr":"172.21.15.96:6811/32072","heartbeat_back_addr":"172.21.15.96:6815/32072","heartbeat_front_addr":"172.21.15.96:6813/32072","state":["exists","up"]},{"osd":2,"uuid":"a173321f-4479-41a3-a3d1-33558b8ec04d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6816","nonce":32078},{"type":"v1","addr":"172.21.15.96:6817","nonce":32078}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6818","nonce":32078},{"type":"v1","addr":"172.21.15.96:6819","nonce":32078}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6822","nonce":32078},{"type":"v1","addr":"172.21.15.96:6823","nonce":32078}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.96:6820","nonce":32078},{"type":"v1","addr":"172.21.15.96:6821","nonce":32078}]},"public_addr":"172.21.15.96:6817/32078","cluster_addr":"172.21.15.96:6819/32078","heartbeat_back_addr":"172.21.15.96:6823/32078","heartbeat_front_addr":"172.21.15.96:6821/32078","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:29.669596+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:29.800709+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:29.855947+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2021-11-11T20:56:35.603 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.0 flush_pg_stats 2021-11-11T20:56:35.750 INFO:teuthology.orchestra.run.smithi096.stdout:34359738371 2021-11-11T20:56:35.750 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.1 flush_pg_stats 2021-11-11T20:56:35.908 INFO:teuthology.orchestra.run.smithi096.stdout:38654705667 2021-11-11T20:56:35.908 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 tell osd.2 flush_pg_stats 2021-11-11T20:56:36.071 INFO:teuthology.orchestra.run.smithi096.stdout:38654705668 2021-11-11T20:56:36.071 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.0 2021-11-11T20:56:36.422 INFO:teuthology.orchestra.run.smithi096.stdout:34359738370 2021-11-11T20:56:36.435 INFO:tasks.ceph.ceph_manager.c1:need seq 34359738371 got 34359738370 for osd.0 2021-11-11T20:56:37.436 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.0 2021-11-11T20:56:37.731 INFO:teuthology.orchestra.run.smithi096.stdout:34359738372 2021-11-11T20:56:37.745 INFO:tasks.ceph.ceph_manager.c1:need seq 34359738371 got 34359738372 for osd.0 2021-11-11T20:56:37.745 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.1 2021-11-11T20:56:38.024 INFO:teuthology.orchestra.run.smithi096.stdout:38654705667 2021-11-11T20:56:38.037 INFO:tasks.ceph.ceph_manager.c1:need seq 38654705667 got 38654705667 for osd.1 2021-11-11T20:56:38.037 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 osd last-stat-seq osd.2 2021-11-11T20:56:38.316 INFO:teuthology.orchestra.run.smithi096.stdout:38654705668 2021-11-11T20:56:38.329 INFO:tasks.ceph.ceph_manager.c1:need seq 38654705668 got 38654705668 for osd.2 2021-11-11T20:56:38.329 INFO:tasks.ceph.ceph_manager.c1:waiting for clean 2021-11-11T20:56:38.330 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 pg dump --format=json 2021-11-11T20:56:38.602 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:38.604 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2021-11-11T20:56:38.615 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":13,"stamp":"2021-11-11T20:56:37.650726+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":96,"num_read_kb":82,"num_write":123,"num_write_kb":1632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85,"ondisk_log_size":85,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":314572800,"kb_used":22908,"kb_used_data":420,"kb_used_omap":0,"kb_used_meta":22464,"kb_avail":314549892,"statfs":{"total":322122547200,"available":322099089408,"internally_reserved":0,"allocated":430080,"data_stored":206004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":23003136},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.994989"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.660952+0000","last_change":"2021-11-11T20:56:34.656194+0000","last_active":"2021-11-11T20:56:34.660952+0000","last_peered":"2021-11-11T20:56:34.660952+0000","last_clean":"2021-11-11T20:56:34.660952+0000","last_became_active":"2021-11-11T20:56:34.655054+0000","last_became_peered":"2021-11-11T20:56:34.655054+0000","last_unstale":"2021-11-11T20:56:34.660952+0000","last_undegraded":"2021-11-11T20:56:34.660952+0000","last_fullsized":"2021-11-11T20:56:34.660952+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T04:04:43.358250+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.660914+0000","last_change":"2021-11-11T20:56:34.655530+0000","last_active":"2021-11-11T20:56:34.660914+0000","last_peered":"2021-11-11T20:56:34.660914+0000","last_clean":"2021-11-11T20:56:34.660914+0000","last_became_active":"2021-11-11T20:56:34.655201+0000","last_became_peered":"2021-11-11T20:56:34.655201+0000","last_unstale":"2021-11-11T20:56:34.660914+0000","last_undegraded":"2021-11-11T20:56:34.660914+0000","last_fullsized":"2021-11-11T20:56:34.660914+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-12T21:47:05.289446+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.660256+0000","last_change":"2021-11-11T20:56:34.659731+0000","last_active":"2021-11-11T20:56:34.660256+0000","last_peered":"2021-11-11T20:56:34.660256+0000","last_clean":"2021-11-11T20:56:34.660256+0000","last_became_active":"2021-11-11T20:56:34.659384+0000","last_became_peered":"2021-11-11T20:56:34.659384+0000","last_unstale":"2021-11-11T20:56:34.660256+0000","last_undegraded":"2021-11-11T20:56:34.660256+0000","last_fullsized":"2021-11-11T20:56:34.660256+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T00:31:23.481020+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.660938+0000","last_change":"2021-11-11T20:56:34.656020+0000","last_active":"2021-11-11T20:56:34.660938+0000","last_peered":"2021-11-11T20:56:34.660938+0000","last_clean":"2021-11-11T20:56:34.660938+0000","last_became_active":"2021-11-11T20:56:34.655746+0000","last_became_peered":"2021-11-11T20:56:34.655746+0000","last_unstale":"2021-11-11T20:56:34.660938+0000","last_undegraded":"2021-11-11T20:56:34.660938+0000","last_fullsized":"2021-11-11T20:56:34.660938+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T06:40:08.538392+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":8,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.657999+0000","last_change":"2021-11-11T20:56:34.657999+0000","last_active":"2021-11-11T20:56:34.657999+0000","last_peered":"2021-11-11T20:56:34.657999+0000","last_clean":"2021-11-11T20:56:34.657999+0000","last_became_active":"2021-11-11T20:56:34.657662+0000","last_became_peered":"2021-11-11T20:56:34.657662+0000","last_unstale":"2021-11-11T20:56:34.657999+0000","last_undegraded":"2021-11-11T20:56:34.657999+0000","last_fullsized":"2021-11-11T20:56:34.657999+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T08:36:28.938715+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"12'85","reported_seq":120,"reported_epoch":12,"state":"active+clean","last_fresh":"2021-11-11T20:56:36.044545+0000","last_change":"2021-11-11T20:56:35.826469+0000","last_active":"2021-11-11T20:56:36.044545+0000","last_peered":"2021-11-11T20:56:36.044545+0000","last_clean":"2021-11-11T20:56:36.044545+0000","last_became_active":"2021-11-11T20:56:35.825578+0000","last_became_peered":"2021-11-11T20:56:35.825578+0000","last_unstale":"2021-11-11T20:56:36.044545+0000","last_undegraded":"2021-11-11T20:56:36.044545+0000","last_fullsized":"2021-11-11T20:56:36.044545+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:34.641957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:34.641957+0000","last_clean_scrub_stamp":"2021-11-11T20:56:34.641957+0000","log_size":85,"ondisk_log_size":85,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T00:14:41.236189+0000","stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":96,"num_read_kb":82,"num_write":123,"num_write_kb":1632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.660956+0000","last_change":"2021-11-11T20:56:34.656216+0000","last_active":"2021-11-11T20:56:34.660956+0000","last_peered":"2021-11-11T20:56:34.660956+0000","last_clean":"2021-11-11T20:56:34.660956+0000","last_became_active":"2021-11-11T20:56:34.655953+0000","last_became_peered":"2021-11-11T20:56:34.655953+0000","last_unstale":"2021-11-11T20:56:34.660956+0000","last_undegraded":"2021-11-11T20:56:34.660956+0000","last_fullsized":"2021-11-11T20:56:34.660956+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-12T23:27:39.485164+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.661266+0000","last_change":"2021-11-11T20:56:34.658531+0000","last_active":"2021-11-11T20:56:34.661266+0000","last_peered":"2021-11-11T20:56:34.661266+0000","last_clean":"2021-11-11T20:56:34.661266+0000","last_became_active":"2021-11-11T20:56:34.658229+0000","last_became_peered":"2021-11-11T20:56:34.658229+0000","last_unstale":"2021-11-11T20:56:34.661266+0000","last_undegraded":"2021-11-11T20:56:34.661266+0000","last_fullsized":"2021-11-11T20:56:34.661266+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T07:21:09.993237+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.659999+0000","last_change":"2021-11-11T20:56:34.659482+0000","last_active":"2021-11-11T20:56:34.659999+0000","last_peered":"2021-11-11T20:56:34.659999+0000","last_clean":"2021-11-11T20:56:34.659999+0000","last_became_active":"2021-11-11T20:56:34.659154+0000","last_became_peered":"2021-11-11T20:56:34.659154+0000","last_unstale":"2021-11-11T20:56:34.659999+0000","last_undegraded":"2021-11-11T20:56:34.659999+0000","last_fullsized":"2021-11-11T20:56:34.659999+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T02:42:29.287525+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":96,"num_read_kb":82,"num_write":123,"num_write_kb":1632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":860160,"data_stored":853040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85,"ondisk_log_size":85,"up":2,"acting":2,"num_store_stats":2},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7728,"kb_used_data":232,"kb_used_omap":0,"kb_used_meta":7488,"kb_avail":104849872,"statfs":{"total":107374182400,"available":107366268928,"internally_reserved":0,"allocated":237568,"data_stored":158706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7667712},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79300000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76200000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7592,"kb_used_data":96,"kb_used_omap":0,"kb_used_meta":7488,"kb_avail":104850008,"statfs":{"total":107374182400,"available":107366408192,"internally_reserved":0,"allocated":98304,"data_stored":23988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7667712},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.628}]}]},{"osd":1,"up_from":9,"seq":38654705667,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7588,"kb_used_data":92,"kb_used_omap":0,"kb_used_meta":7488,"kb_avail":104850012,"statfs":{"total":107374182400,"available":107366412288,"internally_reserved":0,"allocated":94208,"data_stored":23310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7667712},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54200000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2021-11-11T20:56:38.617 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 pg dump --format=json 2021-11-11T20:56:38.891 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:38.893 INFO:teuthology.orchestra.run.smithi096.stderr:dumped all 2021-11-11T20:56:38.904 INFO:teuthology.orchestra.run.smithi096.stdout:{"pg_ready":true,"pg_map":{"version":13,"stamp":"2021-11-11T20:56:37.650726+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":96,"num_read_kb":82,"num_write":123,"num_write_kb":1632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85,"ondisk_log_size":85,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":314572800,"kb_used":22908,"kb_used_data":420,"kb_used_omap":0,"kb_used_meta":22464,"kb_avail":314549892,"statfs":{"total":322122547200,"available":322099089408,"internally_reserved":0,"allocated":430080,"data_stored":206004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":23003136},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.994989"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.660952+0000","last_change":"2021-11-11T20:56:34.656194+0000","last_active":"2021-11-11T20:56:34.660952+0000","last_peered":"2021-11-11T20:56:34.660952+0000","last_clean":"2021-11-11T20:56:34.660952+0000","last_became_active":"2021-11-11T20:56:34.655054+0000","last_became_peered":"2021-11-11T20:56:34.655054+0000","last_unstale":"2021-11-11T20:56:34.660952+0000","last_undegraded":"2021-11-11T20:56:34.660952+0000","last_fullsized":"2021-11-11T20:56:34.660952+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T04:04:43.358250+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.660914+0000","last_change":"2021-11-11T20:56:34.655530+0000","last_active":"2021-11-11T20:56:34.660914+0000","last_peered":"2021-11-11T20:56:34.660914+0000","last_clean":"2021-11-11T20:56:34.660914+0000","last_became_active":"2021-11-11T20:56:34.655201+0000","last_became_peered":"2021-11-11T20:56:34.655201+0000","last_unstale":"2021-11-11T20:56:34.660914+0000","last_undegraded":"2021-11-11T20:56:34.660914+0000","last_fullsized":"2021-11-11T20:56:34.660914+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-12T21:47:05.289446+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.660256+0000","last_change":"2021-11-11T20:56:34.659731+0000","last_active":"2021-11-11T20:56:34.660256+0000","last_peered":"2021-11-11T20:56:34.660256+0000","last_clean":"2021-11-11T20:56:34.660256+0000","last_became_active":"2021-11-11T20:56:34.659384+0000","last_became_peered":"2021-11-11T20:56:34.659384+0000","last_unstale":"2021-11-11T20:56:34.660256+0000","last_undegraded":"2021-11-11T20:56:34.660256+0000","last_fullsized":"2021-11-11T20:56:34.660256+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T00:31:23.481020+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.660938+0000","last_change":"2021-11-11T20:56:34.656020+0000","last_active":"2021-11-11T20:56:34.660938+0000","last_peered":"2021-11-11T20:56:34.660938+0000","last_clean":"2021-11-11T20:56:34.660938+0000","last_became_active":"2021-11-11T20:56:34.655746+0000","last_became_peered":"2021-11-11T20:56:34.655746+0000","last_unstale":"2021-11-11T20:56:34.660938+0000","last_undegraded":"2021-11-11T20:56:34.660938+0000","last_fullsized":"2021-11-11T20:56:34.660938+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T06:40:08.538392+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":8,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.657999+0000","last_change":"2021-11-11T20:56:34.657999+0000","last_active":"2021-11-11T20:56:34.657999+0000","last_peered":"2021-11-11T20:56:34.657999+0000","last_clean":"2021-11-11T20:56:34.657999+0000","last_became_active":"2021-11-11T20:56:34.657662+0000","last_became_peered":"2021-11-11T20:56:34.657662+0000","last_unstale":"2021-11-11T20:56:34.657999+0000","last_undegraded":"2021-11-11T20:56:34.657999+0000","last_fullsized":"2021-11-11T20:56:34.657999+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T08:36:28.938715+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"12'85","reported_seq":120,"reported_epoch":12,"state":"active+clean","last_fresh":"2021-11-11T20:56:36.044545+0000","last_change":"2021-11-11T20:56:35.826469+0000","last_active":"2021-11-11T20:56:36.044545+0000","last_peered":"2021-11-11T20:56:36.044545+0000","last_clean":"2021-11-11T20:56:36.044545+0000","last_became_active":"2021-11-11T20:56:35.825578+0000","last_became_peered":"2021-11-11T20:56:35.825578+0000","last_unstale":"2021-11-11T20:56:36.044545+0000","last_undegraded":"2021-11-11T20:56:36.044545+0000","last_fullsized":"2021-11-11T20:56:36.044545+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:34.641957+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:34.641957+0000","last_clean_scrub_stamp":"2021-11-11T20:56:34.641957+0000","log_size":85,"ondisk_log_size":85,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T00:14:41.236189+0000","stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":96,"num_read_kb":82,"num_write":123,"num_write_kb":1632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.660956+0000","last_change":"2021-11-11T20:56:34.656216+0000","last_active":"2021-11-11T20:56:34.660956+0000","last_peered":"2021-11-11T20:56:34.660956+0000","last_clean":"2021-11-11T20:56:34.660956+0000","last_became_active":"2021-11-11T20:56:34.655953+0000","last_became_peered":"2021-11-11T20:56:34.655953+0000","last_unstale":"2021-11-11T20:56:34.660956+0000","last_undegraded":"2021-11-11T20:56:34.660956+0000","last_fullsized":"2021-11-11T20:56:34.660956+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-12T23:27:39.485164+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.661266+0000","last_change":"2021-11-11T20:56:34.658531+0000","last_active":"2021-11-11T20:56:34.661266+0000","last_peered":"2021-11-11T20:56:34.661266+0000","last_clean":"2021-11-11T20:56:34.661266+0000","last_became_active":"2021-11-11T20:56:34.658229+0000","last_became_peered":"2021-11-11T20:56:34.658229+0000","last_unstale":"2021-11-11T20:56:34.661266+0000","last_undegraded":"2021-11-11T20:56:34.661266+0000","last_fullsized":"2021-11-11T20:56:34.661266+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T07:21:09.993237+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":9,"reported_epoch":11,"state":"active+clean","last_fresh":"2021-11-11T20:56:34.659999+0000","last_change":"2021-11-11T20:56:34.659482+0000","last_active":"2021-11-11T20:56:34.659999+0000","last_peered":"2021-11-11T20:56:34.659999+0000","last_clean":"2021-11-11T20:56:34.659999+0000","last_became_active":"2021-11-11T20:56:34.659154+0000","last_became_peered":"2021-11-11T20:56:34.659154+0000","last_unstale":"2021-11-11T20:56:34.659999+0000","last_undegraded":"2021-11-11T20:56:34.659999+0000","last_fullsized":"2021-11-11T20:56:34.659999+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:33.636566+0000","last_clean_scrub_stamp":"2021-11-11T20:56:33.636566+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T02:42:29.287525+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":96,"num_read_kb":82,"num_write":123,"num_write_kb":1632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":860160,"data_stored":853040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85,"ondisk_log_size":85,"up":2,"acting":2,"num_store_stats":2},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7728,"kb_used_data":232,"kb_used_omap":0,"kb_used_meta":7488,"kb_avail":104849872,"statfs":{"total":107374182400,"available":107366268928,"internally_reserved":0,"allocated":237568,"data_stored":158706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7667712},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79300000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76200000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7592,"kb_used_data":96,"kb_used_omap":0,"kb_used_meta":7488,"kb_avail":104850008,"statfs":{"total":107374182400,"available":107366408192,"internally_reserved":0,"allocated":98304,"data_stored":23988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7667712},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.628}]}]},{"osd":1,"up_from":9,"seq":38654705667,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7588,"kb_used_data":92,"kb_used_omap":0,"kb_used_meta":7488,"kb_avail":104850012,"statfs":{"total":107374182400,"available":107366412288,"internally_reserved":0,"allocated":94208,"data_stored":23310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7667712},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54200000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2021-11-11T20:56:38.905 INFO:tasks.ceph.ceph_manager.c1:clean! 2021-11-11T20:56:38.906 INFO:tasks.ceph:Waiting until ceph cluster c1 is healthy... 2021-11-11T20:56:38.906 INFO:tasks.ceph.ceph_manager.c1:wait_until_healthy 2021-11-11T20:56:38.906 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c1 health --format=json 2021-11-11T20:56:39.218 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:39.218 INFO:teuthology.orchestra.run.smithi096.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2021-11-11T20:56:39.232 INFO:tasks.ceph.ceph_manager.c1:wait_until_healthy done 2021-11-11T20:56:39.232 INFO:teuthology.run_tasks:Running task ceph... 2021-11-11T20:56:39.244 INFO:tasks.ceph:Creating ceph cluster c2... 2021-11-11T20:56:39.244 INFO:tasks.ceph:config {'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug filestore': 20, 'debug journal': 20, 'debug ms': 20, 'debug osd': 25, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)'], 'cpu_profile': set(), 'cluster': 'c2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2021-11-11T20:56:39.244 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/soumyakoduri-2021-11-11_20:19:05-rgw-cloudtiering-distro-basic-smithi/6497986', 'branch': 'cloudtiering', 'description': 'rgw/multisite/{clusters frontend/beast ignore-pg-availability omap_limits overrides realms/three-zone-plus-pubsub supported-random-distro$/{rhel_8} tasks/test_multi}', 'email': None, 'first_in_suite': False, 'job_id': '6497986', 'kernel': {'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'name': 'soumyakoduri-2021-11-11_20:19:05-rgw-cloudtiering-distro-basic-smithi', 'nuke-on-error': True, 'os_type': 'rhel', 'os_version': '8.4', 'overrides': {'admin_socket': {'branch': 'cloudtiering'}, 'ceph': {'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug filestore': 20, 'debug journal': 20, 'debug ms': 20, 'debug osd': 25, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)'], 'log-whitelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098', 'wait-for-scrub': False}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {'osd default pool size': 2}}}, 'install': {'ceph': {'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098'}}, 'rgw': {'compression type': 'random', 'frontend': 'beast'}, 'rgw-multisite': {'realm': {'is default': True, 'name': 'test-realm'}, 'zonegroups': [{'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'test-zonegroup', 'zones': [{'endpoints': ['c1.client.0'], 'is_default': True, 'is_master': True, 'name': 'test-zone1'}, {'endpoints': ['c2.client.0'], 'is_default': True, 'name': 'test-zone2'}, {'endpoints': ['c1.client.1'], 'name': 'test-zone3'}, {'endpoints': ['c2.client.1'], 'is_pubsub': True, 'name': 'test-zone4'}]}]}, 'rgw-multisite-tests': {'args': ['tests.py', 'tests_ps.py']}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'cloudtiering', 'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098'}}, 'owner': 'scheduled_soumyakoduri@teuthology', 'priority': 100, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['c1.mon.a', 'c1.mgr.x', 'c1.osd.0', 'c1.osd.1', 'c1.osd.2', 'c1.client.0', 'c1.client.1'], ['c2.mon.a', 'c2.mgr.x', 'c2.osd.0', 'c2.osd.1', 'c2.osd.2', 'c2.client.0', 'c2.client.1']], 'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098', 'sleep_before_teardown': 0, 'suite': 'rgw', 'suite_branch': 'cloudtiering', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph-c_a3a8018ba3cfe536252b397411bf18b7828c6098/qa', 'suite_relpath': 'qa', 'suite_repo': 'git://git.ceph.com/ceph-ci.git', 'suite_sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098', 'targets': {'smithi096.front.sepia.ceph.com': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCnnzgfZwb9DMmAqSFD0JEkotGtIng9gnT9CmKOgT3btn8RwZTgGqnrdK9xImGAZB25OR1Jma+qf1JWFmOpKBb+PipPVc6yGHbnCX0CLNS4vhkTTvQuBCxnu6Q6MOru6+5yzF3S3PK/bP24It0Crezuq6il9HCWMOf4m7AVYWp867OSYY+x29gpa58Ag2JcBysrC+LPPT0q0k60P6QGLgsp0mIxDEQUfx5e85+dWseIvmN5VzbOeSzuDMRuMr0GcqqTDJSnPI87JzequKN8CrYN6uXtyKZgvj/or7v1ycHdIWFn5MYz/eWRvv5Z5I0mlt8mSLr8U889gsVhH9kwe/e4I/DRdwjBY4WLRbtQe2YjTsTocusQdUwrHwSHuDyvQ/xj/GzvM7OxCI8E8Tq9n2bNWVB33r647D+QJxe42FioRqbdW8QBMAHA2ciBPK6bJ4iuL8AvtPJeD4GhoWgxpZRnrzgu0Xm7+MzcDJeoJ+3ygOrTHprj6AHz+0JMZoFtZEs=', 'smithi198.front.sepia.ceph.com': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9jvdzDl2KeYL6DgYVb9e9leVYHjq3W9lqcBtDftr7EXr6wf0lNmnP+FikYaJOKq8PtBzr50l1o4lYLSSUhsZKS7GwWH1E1T95iY9FYTjv6M3LamGEygDOl905Y1OkkTmcrSWndwQIFh7yJnUVGUzhJxeDMICBoBclIsXjxAHYm4EYaePhsyZ3P6zAhIld/fpRSR3xgN5ggGmryOIHi+GHnN7qTYv+Qso1/vRKIcnFw7Q8GbRbx0wSb6sLwCEL4sA5O4CfLMmVsomQeD8wU8ze9voK/060tkeQtOo77ZTC8UEavnElU4Q/p85M7RQdiIf/lcJh7yN8NkieVh019SiiLPkgZ8D+rtrQsiV6vW5zv3VfC38cIi+l8rilwYomn/r4u6CJc0Og1bIp9yZLcS30RsoShNFxTLNjfBKYNz9IyFFfxtTZyDcPtGwxuZgQyfRhmhJJSwZlOMM7zjbHUB15z/l86+/pT9NYWX0WmHuwRXlQzUwAky053tlXYH0Ui4M='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': None}, {'ceph': {'cluster': 'c1', 'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug filestore': 20, 'debug journal': 20, 'debug ms': 20, 'debug osd': 25, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)'], 'log-whitelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098', 'wait-for-scrub': False}}, {'ceph': {'cluster': 'c2', 'conf': {'client': {'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug filestore': 20, 'debug journal': 20, 'debug ms': 20, 'debug osd': 25, 'rocksdb delete range threshold': 0}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}}, 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(PG_AVAILABILITY\\)'], 'log-whitelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'a3a8018ba3cfe536252b397411bf18b7828c6098', 'wait-for-scrub': False}}, {'rgw': {'c1.client.0': {'port': 8000}, 'c1.client.1': {'port': 8001}, 'c2.client.0': {'port': 8000}, 'c2.client.1': {'port': 8001}}}, {'rgw-multisite': None}, {'rgw-multisite-tests': {'config': {'reconfigure_delay': 60}}}], 'teuthology_branch': 'master', 'timestamp': '2021-11-11_20:19:05', 'tube': 'smithi', 'user': 'soumyakoduri', 'verbose': True, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.20261'} 2021-11-11T20:56:39.245 DEBUG:teuthology.orchestra.run.smithi096:> install -d -m0755 -- /home/ubuntu/cephtest/c2.data 2021-11-11T20:56:39.248 DEBUG:teuthology.orchestra.run.smithi198:> install -d -m0755 -- /home/ubuntu/cephtest/c2.data 2021-11-11T20:56:39.265 DEBUG:teuthology.orchestra.run.smithi096:> sudo install -d -m0777 -- /var/run/ceph 2021-11-11T20:56:39.302 DEBUG:teuthology.orchestra.run.smithi198:> sudo install -d -m0777 -- /var/run/ceph 2021-11-11T20:56:39.334 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:39.334 DEBUG:teuthology.orchestra.run.smithi198:> dd if=/scratch_devs of=/dev/stdout 2021-11-11T20:56:39.391 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2021-11-11T20:56:39.392 DEBUG:teuthology.orchestra.run.smithi198:> stat /dev/vg_nvme/lv_1 && sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_1 2021-11-11T20:56:39.447 INFO:teuthology.orchestra.run.smithi198.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2021-11-11T20:56:39.448 INFO:teuthology.orchestra.run.smithi198.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2021-11-11T20:56:39.448 INFO:teuthology.orchestra.run.smithi198.stdout:Device: 6h/6d Inode: 116151 Links: 1 2021-11-11T20:56:39.448 INFO:teuthology.orchestra.run.smithi198.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2021-11-11T20:56:39.449 INFO:teuthology.orchestra.run.smithi198.stdout:Context: system_u:object_r:device_t:s0 2021-11-11T20:56:39.449 INFO:teuthology.orchestra.run.smithi198.stdout:Access: 2021-11-11 20:55:18.429274079 +0000 2021-11-11T20:56:39.449 INFO:teuthology.orchestra.run.smithi198.stdout:Modify: 2021-11-11 20:45:09.173250647 +0000 2021-11-11T20:56:39.449 INFO:teuthology.orchestra.run.smithi198.stdout:Change: 2021-11-11 20:45:09.173250647 +0000 2021-11-11T20:56:39.449 INFO:teuthology.orchestra.run.smithi198.stdout: Birth: - 2021-11-11T20:56:39.458 INFO:teuthology.orchestra.run.smithi198.stderr:1+0 records in 2021-11-11T20:56:39.459 INFO:teuthology.orchestra.run.smithi198.stderr:1+0 records out 2021-11-11T20:56:39.459 INFO:teuthology.orchestra.run.smithi198.stderr:512 bytes copied, 0.000227416 s, 2.3 MB/s 2021-11-11T20:56:39.462 DEBUG:teuthology.orchestra.run.smithi198:> stat /dev/vg_nvme/lv_2 && sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_2 2021-11-11T20:56:39.517 INFO:teuthology.orchestra.run.smithi198.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2021-11-11T20:56:39.517 INFO:teuthology.orchestra.run.smithi198.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2021-11-11T20:56:39.517 INFO:teuthology.orchestra.run.smithi198.stdout:Device: 6h/6d Inode: 118867 Links: 1 2021-11-11T20:56:39.518 INFO:teuthology.orchestra.run.smithi198.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2021-11-11T20:56:39.518 INFO:teuthology.orchestra.run.smithi198.stdout:Context: system_u:object_r:device_t:s0 2021-11-11T20:56:39.518 INFO:teuthology.orchestra.run.smithi198.stdout:Access: 2021-11-11 20:55:18.430274059 +0000 2021-11-11T20:56:39.518 INFO:teuthology.orchestra.run.smithi198.stdout:Modify: 2021-11-11 20:45:09.563242844 +0000 2021-11-11T20:56:39.519 INFO:teuthology.orchestra.run.smithi198.stdout:Change: 2021-11-11 20:45:09.563242844 +0000 2021-11-11T20:56:39.519 INFO:teuthology.orchestra.run.smithi198.stdout: Birth: - 2021-11-11T20:56:39.527 INFO:teuthology.orchestra.run.smithi198.stderr:1+0 records in 2021-11-11T20:56:39.527 INFO:teuthology.orchestra.run.smithi198.stderr:1+0 records out 2021-11-11T20:56:39.528 INFO:teuthology.orchestra.run.smithi198.stderr:512 bytes copied, 0.000221348 s, 2.3 MB/s 2021-11-11T20:56:39.531 DEBUG:teuthology.orchestra.run.smithi198:> stat /dev/vg_nvme/lv_3 && sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_3 2021-11-11T20:56:39.586 INFO:teuthology.orchestra.run.smithi198.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2021-11-11T20:56:39.587 INFO:teuthology.orchestra.run.smithi198.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2021-11-11T20:56:39.587 INFO:teuthology.orchestra.run.smithi198.stdout:Device: 6h/6d Inode: 110533 Links: 1 2021-11-11T20:56:39.587 INFO:teuthology.orchestra.run.smithi198.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2021-11-11T20:56:39.588 INFO:teuthology.orchestra.run.smithi198.stdout:Context: system_u:object_r:device_t:s0 2021-11-11T20:56:39.588 INFO:teuthology.orchestra.run.smithi198.stdout:Access: 2021-11-11 20:55:18.430274059 +0000 2021-11-11T20:56:39.588 INFO:teuthology.orchestra.run.smithi198.stdout:Modify: 2021-11-11 20:45:09.900236107 +0000 2021-11-11T20:56:39.588 INFO:teuthology.orchestra.run.smithi198.stdout:Change: 2021-11-11 20:45:09.900236107 +0000 2021-11-11T20:56:39.588 INFO:teuthology.orchestra.run.smithi198.stdout: Birth: - 2021-11-11T20:56:39.597 INFO:teuthology.orchestra.run.smithi198.stderr:1+0 records in 2021-11-11T20:56:39.597 INFO:teuthology.orchestra.run.smithi198.stderr:1+0 records out 2021-11-11T20:56:39.597 INFO:teuthology.orchestra.run.smithi198.stderr:512 bytes copied, 0.000302974 s, 1.7 MB/s 2021-11-11T20:56:39.600 DEBUG:teuthology.orchestra.run.smithi198:> stat /dev/vg_nvme/lv_4 && sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_4 2021-11-11T20:56:39.654 INFO:teuthology.orchestra.run.smithi198.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2021-11-11T20:56:39.654 INFO:teuthology.orchestra.run.smithi198.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2021-11-11T20:56:39.655 INFO:teuthology.orchestra.run.smithi198.stdout:Device: 6h/6d Inode: 119881 Links: 1 2021-11-11T20:56:39.655 INFO:teuthology.orchestra.run.smithi198.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2021-11-11T20:56:39.655 INFO:teuthology.orchestra.run.smithi198.stdout:Context: system_u:object_r:device_t:s0 2021-11-11T20:56:39.655 INFO:teuthology.orchestra.run.smithi198.stdout:Access: 2021-11-11 20:55:18.430274059 +0000 2021-11-11T20:56:39.656 INFO:teuthology.orchestra.run.smithi198.stdout:Modify: 2021-11-11 20:45:10.232229468 +0000 2021-11-11T20:56:39.656 INFO:teuthology.orchestra.run.smithi198.stdout:Change: 2021-11-11 20:45:10.232229468 +0000 2021-11-11T20:56:39.656 INFO:teuthology.orchestra.run.smithi198.stdout: Birth: - 2021-11-11T20:56:39.664 INFO:teuthology.orchestra.run.smithi198.stderr:1+0 records in 2021-11-11T20:56:39.665 INFO:teuthology.orchestra.run.smithi198.stderr:1+0 records out 2021-11-11T20:56:39.665 INFO:teuthology.orchestra.run.smithi198.stderr:512 bytes copied, 0.00018857 s, 2.7 MB/s 2021-11-11T20:56:39.667 INFO:tasks.ceph:osd dev map: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2021-11-11T20:56:39.668 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi198.front.sepia.ceph.com'): {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'}} 2021-11-11T20:56:39.668 INFO:tasks.ceph:Generating config... 2021-11-11T20:56:39.669 INFO:tasks.ceph:[client] debug rgw = 20 2021-11-11T20:56:39.669 INFO:tasks.ceph:[client] rgw crypt require ssl = False 2021-11-11T20:56:39.670 INFO:tasks.ceph:[client] rgw crypt s3 kms backend = testing 2021-11-11T20:56:39.670 INFO:tasks.ceph:[client] rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2021-11-11T20:56:39.670 INFO:tasks.ceph:[client] rgw curl low speed time = 300 2021-11-11T20:56:39.671 INFO:tasks.ceph:[client] rgw data log num shards = 4 2021-11-11T20:56:39.671 INFO:tasks.ceph:[client] rgw md log max shards = 4 2021-11-11T20:56:39.671 INFO:tasks.ceph:[client] rgw sync log trim interval = 0 2021-11-11T20:56:39.672 INFO:tasks.ceph:[client] rgw sync meta inject err probability = 0.1 2021-11-11T20:56:39.672 INFO:tasks.ceph:[client] rgw sync obj etag verify = True 2021-11-11T20:56:39.672 INFO:tasks.ceph:[client] setgroup = ceph 2021-11-11T20:56:39.673 INFO:tasks.ceph:[client] setuser = ceph 2021-11-11T20:56:39.673 INFO:tasks.ceph:[mgr] debug mgr = 20 2021-11-11T20:56:39.673 INFO:tasks.ceph:[mgr] debug ms = 1 2021-11-11T20:56:39.674 INFO:tasks.ceph:[mon] debug mon = 20 2021-11-11T20:56:39.674 INFO:tasks.ceph:[mon] debug ms = 1 2021-11-11T20:56:39.674 INFO:tasks.ceph:[mon] debug paxos = 20 2021-11-11T20:56:39.674 INFO:tasks.ceph:[osd] debug filestore = 20 2021-11-11T20:56:39.675 INFO:tasks.ceph:[osd] debug journal = 20 2021-11-11T20:56:39.675 INFO:tasks.ceph:[osd] debug ms = 20 2021-11-11T20:56:39.675 INFO:tasks.ceph:[osd] debug osd = 25 2021-11-11T20:56:39.676 INFO:tasks.ceph:[osd] rocksdb delete range threshold = 0 2021-11-11T20:56:39.676 INFO:tasks.ceph:[osd.0] osd_max_omap_entries_per_request = 10 2021-11-11T20:56:39.676 INFO:tasks.ceph:[osd.1] osd_max_omap_entries_per_request = 1000 2021-11-11T20:56:39.677 INFO:tasks.ceph:[osd.2] osd_max_omap_entries_per_request = 10000 2021-11-11T20:56:39.677 INFO:tasks.ceph:Setting up c2.mon.a... 2021-11-11T20:56:39.677 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/c2.keyring 2021-11-11T20:56:39.755 INFO:teuthology.orchestra.run.smithi198.stdout:creating /etc/ceph/c2.keyring 2021-11-11T20:56:39.758 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/c2.keyring 2021-11-11T20:56:39.796 DEBUG:teuthology.orchestra.run.smithi198:> sudo chmod 0644 /etc/ceph/c2.keyring 2021-11-11T20:56:39.859 DEBUG:tasks.ceph:Ceph mon addresses: [('c2.mon.a', '172.21.15.198')] 2021-11-11T20:56:39.860 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.198'}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'debug filestore': 20, 'debug journal': 20, 'debug ms': 20, 'debug osd': 25, 'rocksdb delete range threshold': 0}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rgw': 20, 'rgw crypt require ssl': False, 'rgw crypt s3 kms backend': 'testing', 'rgw crypt s3 kms encryption keys': 'testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=', 'rgw curl low speed time': 300, 'rgw data log num shards': 4, 'rgw md log max shards': 4, 'rgw sync log trim interval': 0, 'rgw sync meta inject err probability': 0.1, 'rgw sync obj etag verify': True, 'setgroup': 'ceph', 'setuser': 'ceph'}, 'mon.a': {}, 'osd.0': {'osd_max_omap_entries_per_request': 10}, 'osd.1': {'osd_max_omap_entries_per_request': 1000}, 'osd.2': {'osd_max_omap_entries_per_request': 10000}} 2021-11-11T20:56:39.861 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:39.861 DEBUG:teuthology.orchestra.run.smithi198:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2021-11-11T20:56:39.914 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.198 --print /home/ubuntu/cephtest/c2.monmap 2021-11-11T20:56:39.987 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:56:39.988 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:56:39.989 INFO:teuthology.orchestra.run.smithi198.stdout:monmaptool: monmap file /home/ubuntu/cephtest/c2.monmap 2021-11-11T20:56:39.989 INFO:teuthology.orchestra.run.smithi198.stdout:monmaptool: generated fsid e2c5170a-0bbc-4bab-972e-5a438845bdd2 2021-11-11T20:56:39.989 INFO:teuthology.orchestra.run.smithi198.stdout:setting min_mon_release = octopus 2021-11-11T20:56:39.990 INFO:teuthology.orchestra.run.smithi198.stdout:epoch 0 2021-11-11T20:56:39.990 INFO:teuthology.orchestra.run.smithi198.stdout:fsid e2c5170a-0bbc-4bab-972e-5a438845bdd2 2021-11-11T20:56:39.990 INFO:teuthology.orchestra.run.smithi198.stdout:last_changed 2021-11-11T20:56:39.989531+0000 2021-11-11T20:56:39.990 INFO:teuthology.orchestra.run.smithi198.stdout:created 2021-11-11T20:56:39.989531+0000 2021-11-11T20:56:39.991 INFO:teuthology.orchestra.run.smithi198.stdout:min_mon_release 15 (octopus) 2021-11-11T20:56:39.991 INFO:teuthology.orchestra.run.smithi198.stdout:election_strategy: 1 2021-11-11T20:56:39.991 INFO:teuthology.orchestra.run.smithi198.stdout:0: [v2:172.21.15.198:3300/0,v1:172.21.15.198:6789/0] mon.a 2021-11-11T20:56:39.991 INFO:teuthology.orchestra.run.smithi198.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/c2.monmap (1 monitors) 2021-11-11T20:56:39.992 DEBUG:teuthology.orchestra.run.smithi198:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2021-11-11T20:56:40.043 INFO:tasks.ceph:Writing /etc/ceph/c2.conf for FSID e2c5170a-0bbc-4bab-972e-5a438845bdd2... 2021-11-11T20:56:40.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c2.conf && sudo chmod 0644 /etc/ceph/c2.conf > /dev/null 2021-11-11T20:56:40.047 DEBUG:teuthology.orchestra.run.smithi198:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/c2.conf && sudo chmod 0644 /etc/ceph/c2.conf > /dev/null 2021-11-11T20:56:40.095 INFO:teuthology.orchestra.run.smithi096.stdout:[global] 2021-11-11T20:56:40.095 INFO:teuthology.orchestra.run.smithi096.stdout: chdir = "" 2021-11-11T20:56:40.096 INFO:teuthology.orchestra.run.smithi096.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2021-11-11T20:56:40.096 INFO:teuthology.orchestra.run.smithi096.stdout: auth supported = cephx 2021-11-11T20:56:40.096 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.096 INFO:teuthology.orchestra.run.smithi096.stdout: filestore xattr use omap = true 2021-11-11T20:56:40.097 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.097 INFO:teuthology.orchestra.run.smithi096.stdout: mon clock drift allowed = 1.000 2021-11-11T20:56:40.097 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.097 INFO:teuthology.orchestra.run.smithi096.stdout: osd crush chooseleaf type = 0 2021-11-11T20:56:40.098 INFO:teuthology.orchestra.run.smithi096.stdout: auth debug = true 2021-11-11T20:56:40.098 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.098 INFO:teuthology.orchestra.run.smithi096.stdout: ms die on old message = true 2021-11-11T20:56:40.098 INFO:teuthology.orchestra.run.smithi096.stdout: ms die on bug = true 2021-11-11T20:56:40.099 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.099 INFO:teuthology.orchestra.run.smithi096.stdout: mon max pg per osd = 10000 # >= luminous 2021-11-11T20:56:40.099 INFO:teuthology.orchestra.run.smithi096.stdout: mon pg warn max object skew = 0 2021-11-11T20:56:40.099 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.100 INFO:teuthology.orchestra.run.smithi096.stdout: # disable pg_autoscaler by default for new pools 2021-11-11T20:56:40.100 INFO:teuthology.orchestra.run.smithi096.stdout: osd_pool_default_pg_autoscale_mode = off 2021-11-11T20:56:40.100 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.100 INFO:teuthology.orchestra.run.smithi096.stdout: osd pool default size = 2 2021-11-11T20:56:40.100 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.101 INFO:teuthology.orchestra.run.smithi096.stdout: mon osd allow primary affinity = true 2021-11-11T20:56:40.101 INFO:teuthology.orchestra.run.smithi096.stdout: mon osd allow pg remap = true 2021-11-11T20:56:40.101 INFO:teuthology.orchestra.run.smithi096.stdout: mon warn on legacy crush tunables = false 2021-11-11T20:56:40.101 INFO:teuthology.orchestra.run.smithi096.stdout: mon warn on crush straw calc version zero = false 2021-11-11T20:56:40.101 INFO:teuthology.orchestra.run.smithi096.stdout: mon warn on no sortbitwise = false 2021-11-11T20:56:40.102 INFO:teuthology.orchestra.run.smithi096.stdout: mon warn on osd down out interval zero = false 2021-11-11T20:56:40.102 INFO:teuthology.orchestra.run.smithi096.stdout: mon warn on too few osds = false 2021-11-11T20:56:40.102 INFO:teuthology.orchestra.run.smithi096.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2021-11-11T20:56:40.102 INFO:teuthology.orchestra.run.smithi096.stdout: mon_warn_on_pool_no_redundancy = false 2021-11-11T20:56:40.103 INFO:teuthology.orchestra.run.smithi096.stdout: mon_allow_pool_size_one = true 2021-11-11T20:56:40.103 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.103 INFO:teuthology.orchestra.run.smithi096.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2021-11-11T20:56:40.103 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.104 INFO:teuthology.orchestra.run.smithi096.stdout: osd default data pool replay window = 5 2021-11-11T20:56:40.104 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.104 INFO:teuthology.orchestra.run.smithi096.stdout: mon allow pool delete = true 2021-11-11T20:56:40.104 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.105 INFO:teuthology.orchestra.run.smithi096.stdout: mon cluster log file level = debug 2021-11-11T20:56:40.105 INFO:teuthology.orchestra.run.smithi096.stdout: debug asserts on shutdown = true 2021-11-11T20:56:40.105 INFO:teuthology.orchestra.run.smithi096.stdout: mon health detail to clog = false 2021-11-11T20:56:40.105 INFO:teuthology.orchestra.run.smithi096.stdout: mon host = 172.21.15.198 2021-11-11T20:56:40.105 INFO:teuthology.orchestra.run.smithi096.stdout: fsid = e2c5170a-0bbc-4bab-972e-5a438845bdd2 2021-11-11T20:56:40.106 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.106 INFO:teuthology.orchestra.run.smithi096.stdout:[osd] 2021-11-11T20:56:40.106 INFO:teuthology.orchestra.run.smithi096.stdout: osd journal size = 100 2021-11-11T20:56:40.107 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.107 INFO:teuthology.orchestra.run.smithi096.stdout: osd scrub load threshold = 5.0 2021-11-11T20:56:40.107 INFO:teuthology.orchestra.run.smithi096.stdout: osd scrub max interval = 600 2021-11-11T20:56:40.107 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.108 INFO:teuthology.orchestra.run.smithi096.stdout: osd recover clone overlap = true 2021-11-11T20:56:40.108 INFO:teuthology.orchestra.run.smithi096.stdout: osd recovery max chunk = 1048576 2021-11-11T20:56:40.108 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.108 INFO:teuthology.orchestra.run.smithi096.stdout: osd debug shutdown = true 2021-11-11T20:56:40.109 INFO:teuthology.orchestra.run.smithi096.stdout: osd debug op order = true 2021-11-11T20:56:40.109 INFO:teuthology.orchestra.run.smithi096.stdout: osd debug verify stray on activate = true 2021-11-11T20:56:40.109 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.109 INFO:teuthology.orchestra.run.smithi096.stdout: osd open classes on start = true 2021-11-11T20:56:40.109 INFO:teuthology.orchestra.run.smithi096.stdout: osd debug pg log writeout = true 2021-11-11T20:56:40.110 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.110 INFO:teuthology.orchestra.run.smithi096.stdout: osd deep scrub update digest min age = 30 2021-11-11T20:56:40.110 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.110 INFO:teuthology.orchestra.run.smithi096.stdout: osd map max advance = 10 2021-11-11T20:56:40.111 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.111 INFO:teuthology.orchestra.run.smithi096.stdout: journal zero on create = true 2021-11-11T20:56:40.111 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.111 INFO:teuthology.orchestra.run.smithi096.stdout: filestore ondisk finisher threads = 3 2021-11-11T20:56:40.111 INFO:teuthology.orchestra.run.smithi096.stdout: filestore apply finisher threads = 3 2021-11-11T20:56:40.112 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.112 INFO:teuthology.orchestra.run.smithi096.stdout: bdev debug aio = true 2021-11-11T20:56:40.112 INFO:teuthology.orchestra.run.smithi096.stdout: osd debug misdirected ops = true 2021-11-11T20:56:40.112 INFO:teuthology.orchestra.run.smithi096.stdout: debug filestore = 20 2021-11-11T20:56:40.112 INFO:teuthology.orchestra.run.smithi096.stdout: debug journal = 20 2021-11-11T20:56:40.113 INFO:teuthology.orchestra.run.smithi096.stdout: debug ms = 20 2021-11-11T20:56:40.113 INFO:teuthology.orchestra.run.smithi096.stdout: debug osd = 25 2021-11-11T20:56:40.113 INFO:teuthology.orchestra.run.smithi096.stdout: rocksdb delete range threshold = 0 2021-11-11T20:56:40.113 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.114 INFO:teuthology.orchestra.run.smithi096.stdout:[mgr] 2021-11-11T20:56:40.114 INFO:teuthology.orchestra.run.smithi096.stdout: debug ms = 1 2021-11-11T20:56:40.114 INFO:teuthology.orchestra.run.smithi096.stdout: debug mgr = 20 2021-11-11T20:56:40.114 INFO:teuthology.orchestra.run.smithi096.stdout: debug mon = 20 2021-11-11T20:56:40.115 INFO:teuthology.orchestra.run.smithi096.stdout: debug auth = 20 2021-11-11T20:56:40.115 INFO:teuthology.orchestra.run.smithi096.stdout: mon reweight min pgs per osd = 4 2021-11-11T20:56:40.115 INFO:teuthology.orchestra.run.smithi096.stdout: mon reweight min bytes per osd = 10 2021-11-11T20:56:40.115 INFO:teuthology.orchestra.run.smithi096.stdout: mgr/telemetry/nag = false 2021-11-11T20:56:40.115 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.116 INFO:teuthology.orchestra.run.smithi096.stdout:[mon] 2021-11-11T20:56:40.116 INFO:teuthology.orchestra.run.smithi096.stdout: debug ms = 1 2021-11-11T20:56:40.116 INFO:teuthology.orchestra.run.smithi096.stdout: debug mon = 20 2021-11-11T20:56:40.116 INFO:teuthology.orchestra.run.smithi096.stdout: debug paxos = 20 2021-11-11T20:56:40.117 INFO:teuthology.orchestra.run.smithi096.stdout: debug auth = 20 2021-11-11T20:56:40.117 INFO:teuthology.orchestra.run.smithi096.stdout: mon data avail warn = 5 2021-11-11T20:56:40.117 INFO:teuthology.orchestra.run.smithi096.stdout: mon mgr mkfs grace = 240 2021-11-11T20:56:40.117 INFO:teuthology.orchestra.run.smithi096.stdout: mon reweight min pgs per osd = 4 2021-11-11T20:56:40.117 INFO:teuthology.orchestra.run.smithi096.stdout: mon osd reporter subtree level = osd 2021-11-11T20:56:40.118 INFO:teuthology.orchestra.run.smithi096.stdout: mon osd prime pg temp = true 2021-11-11T20:56:40.118 INFO:teuthology.orchestra.run.smithi096.stdout: mon reweight min bytes per osd = 10 2021-11-11T20:56:40.118 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.119 INFO:teuthology.orchestra.run.smithi096.stdout: # rotate auth tickets quickly to exercise renewal paths 2021-11-11T20:56:40.119 INFO:teuthology.orchestra.run.smithi096.stdout: auth mon ticket ttl = 660 # 11m 2021-11-11T20:56:40.119 INFO:teuthology.orchestra.run.smithi096.stdout: auth service ticket ttl = 240 # 4m 2021-11-11T20:56:40.120 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.120 INFO:teuthology.orchestra.run.smithi096.stdout: # don't complain about insecure global_id in the test suite 2021-11-11T20:56:40.121 INFO:teuthology.orchestra.run.smithi096.stdout: mon_warn_on_insecure_global_id_reclaim = false 2021-11-11T20:56:40.121 INFO:teuthology.orchestra.run.smithi096.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2021-11-11T20:56:40.122 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T20:56:40.122 INFO:teuthology.orchestra.run.smithi096.stdout:[client] 2021-11-11T20:56:40.123 INFO:teuthology.orchestra.run.smithi096.stdout: rgw cache enabled = true 2021-11-11T20:56:40.123 INFO:teuthology.orchestra.run.smithi096.stdout: rgw enable ops log = true 2021-11-11T20:56:40.124 INFO:teuthology.orchestra.run.smithi096.stdout: rgw enable usage log = true 2021-11-11T20:56:40.124 INFO:teuthology.orchestra.run.smithi096.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2021-11-11T20:56:40.125 INFO:teuthology.orchestra.run.smithi096.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2021-11-11T20:56:40.125 INFO:teuthology.orchestra.run.smithi096.stdout: debug rgw = 20 2021-11-11T20:56:40.125 INFO:teuthology.orchestra.run.smithi096.stdout: rgw crypt require ssl = False 2021-11-11T20:56:40.126 INFO:teuthology.orchestra.run.smithi096.stdout: rgw crypt s3 kms backend = testing 2021-11-11T20:56:40.126 INFO:teuthology.orchestra.run.smithi096.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2021-11-11T20:56:40.127 INFO:teuthology.orchestra.run.smithi096.stdout: rgw curl low speed time = 300 2021-11-11T20:56:40.127 INFO:teuthology.orchestra.run.smithi096.stdout: rgw data log num shards = 4 2021-11-11T20:56:40.127 INFO:teuthology.orchestra.run.smithi096.stdout: rgw md log max shards = 4 2021-11-11T20:56:40.128 INFO:teuthology.orchestra.run.smithi096.stdout: rgw sync log trim interval = 0 2021-11-11T20:56:40.128 INFO:teuthology.orchestra.run.smithi096.stdout: rgw sync meta inject err probability = 0.1 2021-11-11T20:56:40.129 INFO:teuthology.orchestra.run.smithi096.stdout: rgw sync obj etag verify = True 2021-11-11T20:56:40.129 INFO:teuthology.orchestra.run.smithi096.stdout: setgroup = ceph 2021-11-11T20:56:40.130 INFO:teuthology.orchestra.run.smithi096.stdout: setuser = ceph 2021-11-11T20:56:40.130 INFO:teuthology.orchestra.run.smithi096.stdout:[mon.a] 2021-11-11T20:56:40.130 INFO:teuthology.orchestra.run.smithi096.stdout:[osd.0] 2021-11-11T20:56:40.131 INFO:teuthology.orchestra.run.smithi096.stdout: osd_max_omap_entries_per_request = 10 2021-11-11T20:56:40.131 INFO:teuthology.orchestra.run.smithi096.stdout:[osd.1] 2021-11-11T20:56:40.132 INFO:teuthology.orchestra.run.smithi096.stdout: osd_max_omap_entries_per_request = 1000 2021-11-11T20:56:40.132 INFO:teuthology.orchestra.run.smithi096.stdout:[osd.2] 2021-11-11T20:56:40.132 INFO:teuthology.orchestra.run.smithi096.stdout: osd_max_omap_entries_per_request = 10000 2021-11-11T20:56:40.134 INFO:teuthology.orchestra.run.smithi198.stdout:[global] 2021-11-11T20:56:40.134 INFO:teuthology.orchestra.run.smithi198.stdout: chdir = "" 2021-11-11T20:56:40.135 INFO:teuthology.orchestra.run.smithi198.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2021-11-11T20:56:40.135 INFO:teuthology.orchestra.run.smithi198.stdout: auth supported = cephx 2021-11-11T20:56:40.135 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.136 INFO:teuthology.orchestra.run.smithi198.stdout: filestore xattr use omap = true 2021-11-11T20:56:40.136 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.136 INFO:teuthology.orchestra.run.smithi198.stdout: mon clock drift allowed = 1.000 2021-11-11T20:56:40.137 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.137 INFO:teuthology.orchestra.run.smithi198.stdout: osd crush chooseleaf type = 0 2021-11-11T20:56:40.138 INFO:teuthology.orchestra.run.smithi198.stdout: auth debug = true 2021-11-11T20:56:40.138 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.139 INFO:teuthology.orchestra.run.smithi198.stdout: ms die on old message = true 2021-11-11T20:56:40.139 INFO:teuthology.orchestra.run.smithi198.stdout: ms die on bug = true 2021-11-11T20:56:40.139 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.140 INFO:teuthology.orchestra.run.smithi198.stdout: mon max pg per osd = 10000 # >= luminous 2021-11-11T20:56:40.140 INFO:teuthology.orchestra.run.smithi198.stdout: mon pg warn max object skew = 0 2021-11-11T20:56:40.141 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.141 INFO:teuthology.orchestra.run.smithi198.stdout: # disable pg_autoscaler by default for new pools 2021-11-11T20:56:40.142 INFO:teuthology.orchestra.run.smithi198.stdout: osd_pool_default_pg_autoscale_mode = off 2021-11-11T20:56:40.142 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.142 INFO:teuthology.orchestra.run.smithi198.stdout: osd pool default size = 2 2021-11-11T20:56:40.143 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.143 INFO:teuthology.orchestra.run.smithi198.stdout: mon osd allow primary affinity = true 2021-11-11T20:56:40.143 INFO:teuthology.orchestra.run.smithi198.stdout: mon osd allow pg remap = true 2021-11-11T20:56:40.144 INFO:teuthology.orchestra.run.smithi198.stdout: mon warn on legacy crush tunables = false 2021-11-11T20:56:40.144 INFO:teuthology.orchestra.run.smithi198.stdout: mon warn on crush straw calc version zero = false 2021-11-11T20:56:40.144 INFO:teuthology.orchestra.run.smithi198.stdout: mon warn on no sortbitwise = false 2021-11-11T20:56:40.145 INFO:teuthology.orchestra.run.smithi198.stdout: mon warn on osd down out interval zero = false 2021-11-11T20:56:40.145 INFO:teuthology.orchestra.run.smithi198.stdout: mon warn on too few osds = false 2021-11-11T20:56:40.146 INFO:teuthology.orchestra.run.smithi198.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2021-11-11T20:56:40.146 INFO:teuthology.orchestra.run.smithi198.stdout: mon_warn_on_pool_no_redundancy = false 2021-11-11T20:56:40.147 INFO:teuthology.orchestra.run.smithi198.stdout: mon_allow_pool_size_one = true 2021-11-11T20:56:40.147 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.148 INFO:teuthology.orchestra.run.smithi198.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2021-11-11T20:56:40.148 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.148 INFO:teuthology.orchestra.run.smithi198.stdout: osd default data pool replay window = 5 2021-11-11T20:56:40.148 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.149 INFO:teuthology.orchestra.run.smithi198.stdout: mon allow pool delete = true 2021-11-11T20:56:40.149 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.149 INFO:teuthology.orchestra.run.smithi198.stdout: mon cluster log file level = debug 2021-11-11T20:56:40.149 INFO:teuthology.orchestra.run.smithi198.stdout: debug asserts on shutdown = true 2021-11-11T20:56:40.150 INFO:teuthology.orchestra.run.smithi198.stdout: mon health detail to clog = false 2021-11-11T20:56:40.150 INFO:teuthology.orchestra.run.smithi198.stdout: mon host = 172.21.15.198 2021-11-11T20:56:40.150 INFO:teuthology.orchestra.run.smithi198.stdout: fsid = e2c5170a-0bbc-4bab-972e-5a438845bdd2 2021-11-11T20:56:40.150 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.150 INFO:teuthology.orchestra.run.smithi198.stdout:[osd] 2021-11-11T20:56:40.151 INFO:teuthology.orchestra.run.smithi198.stdout: osd journal size = 100 2021-11-11T20:56:40.151 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.151 INFO:teuthology.orchestra.run.smithi198.stdout: osd scrub load threshold = 5.0 2021-11-11T20:56:40.151 INFO:teuthology.orchestra.run.smithi198.stdout: osd scrub max interval = 600 2021-11-11T20:56:40.152 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.152 INFO:teuthology.orchestra.run.smithi198.stdout: osd recover clone overlap = true 2021-11-11T20:56:40.152 INFO:teuthology.orchestra.run.smithi198.stdout: osd recovery max chunk = 1048576 2021-11-11T20:56:40.152 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.153 INFO:teuthology.orchestra.run.smithi198.stdout: osd debug shutdown = true 2021-11-11T20:56:40.153 INFO:teuthology.orchestra.run.smithi198.stdout: osd debug op order = true 2021-11-11T20:56:40.153 INFO:teuthology.orchestra.run.smithi198.stdout: osd debug verify stray on activate = true 2021-11-11T20:56:40.153 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.153 INFO:teuthology.orchestra.run.smithi198.stdout: osd open classes on start = true 2021-11-11T20:56:40.154 INFO:teuthology.orchestra.run.smithi198.stdout: osd debug pg log writeout = true 2021-11-11T20:56:40.154 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.154 INFO:teuthology.orchestra.run.smithi198.stdout: osd deep scrub update digest min age = 30 2021-11-11T20:56:40.154 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.155 INFO:teuthology.orchestra.run.smithi198.stdout: osd map max advance = 10 2021-11-11T20:56:40.155 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.155 INFO:teuthology.orchestra.run.smithi198.stdout: journal zero on create = true 2021-11-11T20:56:40.155 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.155 INFO:teuthology.orchestra.run.smithi198.stdout: filestore ondisk finisher threads = 3 2021-11-11T20:56:40.156 INFO:teuthology.orchestra.run.smithi198.stdout: filestore apply finisher threads = 3 2021-11-11T20:56:40.156 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.156 INFO:teuthology.orchestra.run.smithi198.stdout: bdev debug aio = true 2021-11-11T20:56:40.156 INFO:teuthology.orchestra.run.smithi198.stdout: osd debug misdirected ops = true 2021-11-11T20:56:40.157 INFO:teuthology.orchestra.run.smithi198.stdout: debug filestore = 20 2021-11-11T20:56:40.157 INFO:teuthology.orchestra.run.smithi198.stdout: debug journal = 20 2021-11-11T20:56:40.157 INFO:teuthology.orchestra.run.smithi198.stdout: debug ms = 20 2021-11-11T20:56:40.157 INFO:teuthology.orchestra.run.smithi198.stdout: debug osd = 25 2021-11-11T20:56:40.158 INFO:teuthology.orchestra.run.smithi198.stdout: rocksdb delete range threshold = 0 2021-11-11T20:56:40.158 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.158 INFO:teuthology.orchestra.run.smithi198.stdout:[mgr] 2021-11-11T20:56:40.159 INFO:teuthology.orchestra.run.smithi198.stdout: debug ms = 1 2021-11-11T20:56:40.159 INFO:teuthology.orchestra.run.smithi198.stdout: debug mgr = 20 2021-11-11T20:56:40.159 INFO:teuthology.orchestra.run.smithi198.stdout: debug mon = 20 2021-11-11T20:56:40.159 INFO:teuthology.orchestra.run.smithi198.stdout: debug auth = 20 2021-11-11T20:56:40.160 INFO:teuthology.orchestra.run.smithi198.stdout: mon reweight min pgs per osd = 4 2021-11-11T20:56:40.160 INFO:teuthology.orchestra.run.smithi198.stdout: mon reweight min bytes per osd = 10 2021-11-11T20:56:40.160 INFO:teuthology.orchestra.run.smithi198.stdout: mgr/telemetry/nag = false 2021-11-11T20:56:40.160 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.161 INFO:teuthology.orchestra.run.smithi198.stdout:[mon] 2021-11-11T20:56:40.161 INFO:teuthology.orchestra.run.smithi198.stdout: debug ms = 1 2021-11-11T20:56:40.161 INFO:teuthology.orchestra.run.smithi198.stdout: debug mon = 20 2021-11-11T20:56:40.161 INFO:teuthology.orchestra.run.smithi198.stdout: debug paxos = 20 2021-11-11T20:56:40.162 INFO:teuthology.orchestra.run.smithi198.stdout: debug auth = 20 2021-11-11T20:56:40.162 INFO:teuthology.orchestra.run.smithi198.stdout: mon data avail warn = 5 2021-11-11T20:56:40.162 INFO:teuthology.orchestra.run.smithi198.stdout: mon mgr mkfs grace = 240 2021-11-11T20:56:40.162 INFO:teuthology.orchestra.run.smithi198.stdout: mon reweight min pgs per osd = 4 2021-11-11T20:56:40.163 INFO:teuthology.orchestra.run.smithi198.stdout: mon osd reporter subtree level = osd 2021-11-11T20:56:40.163 INFO:teuthology.orchestra.run.smithi198.stdout: mon osd prime pg temp = true 2021-11-11T20:56:40.163 INFO:teuthology.orchestra.run.smithi198.stdout: mon reweight min bytes per osd = 10 2021-11-11T20:56:40.163 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.164 INFO:teuthology.orchestra.run.smithi198.stdout: # rotate auth tickets quickly to exercise renewal paths 2021-11-11T20:56:40.164 INFO:teuthology.orchestra.run.smithi198.stdout: auth mon ticket ttl = 660 # 11m 2021-11-11T20:56:40.164 INFO:teuthology.orchestra.run.smithi198.stdout: auth service ticket ttl = 240 # 4m 2021-11-11T20:56:40.164 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.164 INFO:teuthology.orchestra.run.smithi198.stdout: # don't complain about insecure global_id in the test suite 2021-11-11T20:56:40.165 INFO:teuthology.orchestra.run.smithi198.stdout: mon_warn_on_insecure_global_id_reclaim = false 2021-11-11T20:56:40.165 INFO:teuthology.orchestra.run.smithi198.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2021-11-11T20:56:40.165 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:40.165 INFO:teuthology.orchestra.run.smithi198.stdout:[client] 2021-11-11T20:56:40.166 INFO:teuthology.orchestra.run.smithi198.stdout: rgw cache enabled = true 2021-11-11T20:56:40.166 INFO:teuthology.orchestra.run.smithi198.stdout: rgw enable ops log = true 2021-11-11T20:56:40.166 INFO:teuthology.orchestra.run.smithi198.stdout: rgw enable usage log = true 2021-11-11T20:56:40.166 INFO:teuthology.orchestra.run.smithi198.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2021-11-11T20:56:40.167 INFO:teuthology.orchestra.run.smithi198.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2021-11-11T20:56:40.167 INFO:teuthology.orchestra.run.smithi198.stdout: debug rgw = 20 2021-11-11T20:56:40.167 INFO:teuthology.orchestra.run.smithi198.stdout: rgw crypt require ssl = False 2021-11-11T20:56:40.167 INFO:teuthology.orchestra.run.smithi198.stdout: rgw crypt s3 kms backend = testing 2021-11-11T20:56:40.167 INFO:teuthology.orchestra.run.smithi198.stdout: rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= 2021-11-11T20:56:40.168 INFO:teuthology.orchestra.run.smithi198.stdout: rgw curl low speed time = 300 2021-11-11T20:56:40.168 INFO:teuthology.orchestra.run.smithi198.stdout: rgw data log num shards = 4 2021-11-11T20:56:40.168 INFO:teuthology.orchestra.run.smithi198.stdout: rgw md log max shards = 4 2021-11-11T20:56:40.168 INFO:teuthology.orchestra.run.smithi198.stdout: rgw sync log trim interval = 0 2021-11-11T20:56:40.169 INFO:teuthology.orchestra.run.smithi198.stdout: rgw sync meta inject err probability = 0.1 2021-11-11T20:56:40.169 INFO:teuthology.orchestra.run.smithi198.stdout: rgw sync obj etag verify = True 2021-11-11T20:56:40.169 INFO:teuthology.orchestra.run.smithi198.stdout: setgroup = ceph 2021-11-11T20:56:40.169 INFO:teuthology.orchestra.run.smithi198.stdout: setuser = ceph 2021-11-11T20:56:40.170 INFO:teuthology.orchestra.run.smithi198.stdout:[mon.a] 2021-11-11T20:56:40.170 INFO:teuthology.orchestra.run.smithi198.stdout:[osd.0] 2021-11-11T20:56:40.170 INFO:teuthology.orchestra.run.smithi198.stdout: osd_max_omap_entries_per_request = 10 2021-11-11T20:56:40.171 INFO:teuthology.orchestra.run.smithi198.stdout:[osd.1] 2021-11-11T20:56:40.171 INFO:teuthology.orchestra.run.smithi198.stdout: osd_max_omap_entries_per_request = 1000 2021-11-11T20:56:40.171 INFO:teuthology.orchestra.run.smithi198.stdout:[osd.2] 2021-11-11T20:56:40.171 INFO:teuthology.orchestra.run.smithi198.stdout: osd_max_omap_entries_per_request = 10000 2021-11-11T20:56:40.173 INFO:tasks.ceph:Creating admin key on c2.mon.a... 2021-11-11T20:56:40.173 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/c2.keyring 2021-11-11T20:56:40.265 INFO:tasks.ceph:Copying monmap to all nodes... 2021-11-11T20:56:40.265 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:40.266 DEBUG:teuthology.orchestra.run.smithi198:> dd if=/etc/ceph/c2.keyring of=/dev/stdout 2021-11-11T20:56:40.281 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:40.282 DEBUG:teuthology.orchestra.run.smithi198:> dd if=/home/ubuntu/cephtest/c2.monmap of=/dev/stdout 2021-11-11T20:56:40.335 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi096.front.sepia.ceph.com 2021-11-11T20:56:40.336 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:40.336 DEBUG:teuthology.orchestra.run.smithi096:> sudo dd of=/etc/ceph/c2.keyring 2021-11-11T20:56:40.336 DEBUG:teuthology.orchestra.run.smithi096:> sudo chmod 0644 /etc/ceph/c2.keyring 2021-11-11T20:56:40.376 DEBUG:teuthology.orchestra.run.smithi096:> set -ex 2021-11-11T20:56:40.377 DEBUG:teuthology.orchestra.run.smithi096:> dd of=/home/ubuntu/cephtest/c2.monmap 2021-11-11T20:56:40.432 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi198.front.sepia.ceph.com 2021-11-11T20:56:40.461 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:40.462 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd of=/etc/ceph/c2.keyring 2021-11-11T20:56:40.462 DEBUG:teuthology.orchestra.run.smithi198:> sudo chmod 0644 /etc/ceph/c2.keyring 2021-11-11T20:56:40.495 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:40.495 DEBUG:teuthology.orchestra.run.smithi198:> dd of=/home/ubuntu/cephtest/c2.monmap 2021-11-11T20:56:40.549 INFO:tasks.ceph:Setting up mon nodes... 2021-11-11T20:56:40.550 INFO:tasks.ceph:Setting up mgr nodes... 2021-11-11T20:56:40.550 DEBUG:teuthology.orchestra.run.smithi198:> sudo mkdir -p /var/lib/ceph/mgr/c2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/c2-x/keyring 2021-11-11T20:56:40.634 INFO:teuthology.orchestra.run.smithi198.stdout:creating /var/lib/ceph/mgr/c2-x/keyring 2021-11-11T20:56:40.637 INFO:tasks.ceph:Setting up mds nodes... 2021-11-11T20:56:40.637 INFO:tasks.ceph_client:Setting up client nodes... 2021-11-11T20:56:40.638 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/c2.client.0.keyring && sudo chmod 0644 /etc/ceph/c2.client.0.keyring 2021-11-11T20:56:40.674 INFO:teuthology.orchestra.run.smithi198.stdout:creating /etc/ceph/c2.client.0.keyring 2021-11-11T20:56:40.686 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.1 /etc/ceph/c2.client.1.keyring && sudo chmod 0644 /etc/ceph/c2.client.1.keyring 2021-11-11T20:56:40.763 INFO:teuthology.orchestra.run.smithi198.stdout:creating /etc/ceph/c2.client.1.keyring 2021-11-11T20:56:40.774 INFO:tasks.ceph:Running mkfs on osd nodes... 2021-11-11T20:56:40.775 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi096.front.sepia.ceph.com'): {'c1.osd.0': '/dev/vg_nvme/lv_1', 'c1.osd.1': '/dev/vg_nvme/lv_2', 'c1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@smithi198.front.sepia.ceph.com'): {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'}} 2021-11-11T20:56:40.775 DEBUG:teuthology.orchestra.run.smithi198:> sudo mkdir -p /var/lib/ceph/osd/c2-0 2021-11-11T20:56:40.837 INFO:tasks.ceph:roles_to_devs: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2021-11-11T20:56:40.837 INFO:tasks.ceph:role: c2.osd.0 2021-11-11T20:56:40.838 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi198.front.sepia.ceph.com 2021-11-11T20:56:40.838 DEBUG:teuthology.orchestra.run.smithi198:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2021-11-11T20:56:40.908 INFO:teuthology.orchestra.run.smithi198.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2021-11-11T20:56:40.908 INFO:teuthology.orchestra.run.smithi198.stdout: = sectsz=512 attr=2, projid32bit=1 2021-11-11T20:56:40.909 INFO:teuthology.orchestra.run.smithi198.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2021-11-11T20:56:40.909 INFO:teuthology.orchestra.run.smithi198.stdout: = reflink=1 2021-11-11T20:56:40.909 INFO:teuthology.orchestra.run.smithi198.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2021-11-11T20:56:40.909 INFO:teuthology.orchestra.run.smithi198.stdout: = sunit=0 swidth=0 blks 2021-11-11T20:56:40.909 INFO:teuthology.orchestra.run.smithi198.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2021-11-11T20:56:40.910 INFO:teuthology.orchestra.run.smithi198.stdout:log =internal log bsize=4096 blocks=11443, version=2 2021-11-11T20:56:40.910 INFO:teuthology.orchestra.run.smithi198.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2021-11-11T20:56:40.910 INFO:teuthology.orchestra.run.smithi198.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2021-11-11T20:56:41.271 INFO:teuthology.orchestra.run.smithi198.stdout:Discarding blocks...Done. 2021-11-11T20:56:41.272 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi198.front.sepia.ceph.com -o noatime 2021-11-11T20:56:41.273 DEBUG:teuthology.orchestra.run.smithi198:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/c2-0 2021-11-11T20:56:41.388 DEBUG:teuthology.orchestra.run.smithi198:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-0 2021-11-11T20:56:41.416 DEBUG:teuthology.orchestra.run.smithi198:> sudo mkdir -p /var/lib/ceph/osd/c2-1 2021-11-11T20:56:41.480 INFO:tasks.ceph:roles_to_devs: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2021-11-11T20:56:41.481 INFO:tasks.ceph:role: c2.osd.1 2021-11-11T20:56:41.481 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi198.front.sepia.ceph.com 2021-11-11T20:56:41.481 DEBUG:teuthology.orchestra.run.smithi198:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2021-11-11T20:56:41.551 INFO:teuthology.orchestra.run.smithi198.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2021-11-11T20:56:41.551 INFO:teuthology.orchestra.run.smithi198.stdout: = sectsz=512 attr=2, projid32bit=1 2021-11-11T20:56:41.552 INFO:teuthology.orchestra.run.smithi198.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2021-11-11T20:56:41.552 INFO:teuthology.orchestra.run.smithi198.stdout: = reflink=1 2021-11-11T20:56:41.552 INFO:teuthology.orchestra.run.smithi198.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2021-11-11T20:56:41.552 INFO:teuthology.orchestra.run.smithi198.stdout: = sunit=0 swidth=0 blks 2021-11-11T20:56:41.552 INFO:teuthology.orchestra.run.smithi198.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2021-11-11T20:56:41.553 INFO:teuthology.orchestra.run.smithi198.stdout:log =internal log bsize=4096 blocks=11443, version=2 2021-11-11T20:56:41.553 INFO:teuthology.orchestra.run.smithi198.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2021-11-11T20:56:41.553 INFO:teuthology.orchestra.run.smithi198.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2021-11-11T20:56:41.919 INFO:teuthology.orchestra.run.smithi198.stdout:Discarding blocks...Done. 2021-11-11T20:56:41.920 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi198.front.sepia.ceph.com -o noatime 2021-11-11T20:56:41.921 DEBUG:teuthology.orchestra.run.smithi198:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/c2-1 2021-11-11T20:56:42.005 DEBUG:teuthology.orchestra.run.smithi198:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-1 2021-11-11T20:56:42.032 DEBUG:teuthology.orchestra.run.smithi198:> sudo mkdir -p /var/lib/ceph/osd/c2-2 2021-11-11T20:56:42.096 INFO:tasks.ceph:roles_to_devs: {'c2.osd.0': '/dev/vg_nvme/lv_1', 'c2.osd.1': '/dev/vg_nvme/lv_2', 'c2.osd.2': '/dev/vg_nvme/lv_3'} 2021-11-11T20:56:42.097 INFO:tasks.ceph:role: c2.osd.2 2021-11-11T20:56:42.097 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi198.front.sepia.ceph.com 2021-11-11T20:56:42.098 DEBUG:teuthology.orchestra.run.smithi198:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2021-11-11T20:56:42.167 INFO:teuthology.orchestra.run.smithi198.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2021-11-11T20:56:42.167 INFO:teuthology.orchestra.run.smithi198.stdout: = sectsz=512 attr=2, projid32bit=1 2021-11-11T20:56:42.167 INFO:teuthology.orchestra.run.smithi198.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2021-11-11T20:56:42.168 INFO:teuthology.orchestra.run.smithi198.stdout: = reflink=1 2021-11-11T20:56:42.168 INFO:teuthology.orchestra.run.smithi198.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2021-11-11T20:56:42.168 INFO:teuthology.orchestra.run.smithi198.stdout: = sunit=0 swidth=0 blks 2021-11-11T20:56:42.168 INFO:teuthology.orchestra.run.smithi198.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2021-11-11T20:56:42.168 INFO:teuthology.orchestra.run.smithi198.stdout:log =internal log bsize=4096 blocks=11443, version=2 2021-11-11T20:56:42.169 INFO:teuthology.orchestra.run.smithi198.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2021-11-11T20:56:42.169 INFO:teuthology.orchestra.run.smithi198.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2021-11-11T20:56:42.539 INFO:teuthology.orchestra.run.smithi198.stdout:Discarding blocks...Done. 2021-11-11T20:56:42.540 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi198.front.sepia.ceph.com -o noatime 2021-11-11T20:56:42.540 DEBUG:teuthology.orchestra.run.smithi198:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/c2-2 2021-11-11T20:56:42.635 DEBUG:teuthology.orchestra.run.smithi198:> sudo /sbin/restorecon /var/lib/ceph/osd/c2-2 2021-11-11T20:56:42.663 DEBUG:teuthology.orchestra.run.smithi198:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/c2.monmap 2021-11-11T20:56:42.743 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:42.744+0000 7f3768a340c0 -1 auth: error reading file: /var/lib/ceph/osd/c2-0/keyring: can't open /var/lib/ceph/osd/c2-0/keyring: (2) No such file or directory 2021-11-11T20:56:42.745 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:42.746+0000 7f3768a340c0 -1 created new key in keyring /var/lib/ceph/osd/c2-0/keyring 2021-11-11T20:56:42.746 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:42.746+0000 7f3768a340c0 -1 bluestore(/var/lib/ceph/osd/c2-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-0/block: (2) No such file or directory 2021-11-11T20:56:42.746 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:42.746+0000 7f3768a340c0 -1 bluestore(/var/lib/ceph/osd/c2-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-0/block: (2) No such file or directory 2021-11-11T20:56:42.746 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:42.746+0000 7f3768a340c0 -1 bluestore(/var/lib/ceph/osd/c2-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-0/block: (2) No such file or directory 2021-11-11T20:56:42.750 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:42.750+0000 7f3768a340c0 -1 bluestore(/var/lib/ceph/osd/c2-0) _read_fsid unparsable uuid 2021-11-11T20:56:44.834 DEBUG:teuthology.orchestra.run.smithi198:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-0 2021-11-11T20:56:44.903 DEBUG:teuthology.orchestra.run.smithi198:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/c2.monmap 2021-11-11T20:56:44.989 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:44.989+0000 7fa7c9ee10c0 -1 auth: error reading file: /var/lib/ceph/osd/c2-1/keyring: can't open /var/lib/ceph/osd/c2-1/keyring: (2) No such file or directory 2021-11-11T20:56:44.989 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:44.989+0000 7fa7c9ee10c0 -1 created new key in keyring /var/lib/ceph/osd/c2-1/keyring 2021-11-11T20:56:44.990 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:44.989+0000 7fa7c9ee10c0 -1 bluestore(/var/lib/ceph/osd/c2-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-1/block: (2) No such file or directory 2021-11-11T20:56:44.990 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:44.989+0000 7fa7c9ee10c0 -1 bluestore(/var/lib/ceph/osd/c2-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-1/block: (2) No such file or directory 2021-11-11T20:56:44.990 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:44.989+0000 7fa7c9ee10c0 -1 bluestore(/var/lib/ceph/osd/c2-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-1/block: (2) No such file or directory 2021-11-11T20:56:44.990 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:44.990+0000 7fa7c9ee10c0 -1 bluestore(/var/lib/ceph/osd/c2-1) _read_fsid unparsable uuid 2021-11-11T20:56:47.072 DEBUG:teuthology.orchestra.run.smithi198:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-1 2021-11-11T20:56:47.101 DEBUG:teuthology.orchestra.run.smithi198:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster c2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/c2.monmap 2021-11-11T20:56:47.185 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:47.186+0000 7f91fb0d30c0 -1 auth: error reading file: /var/lib/ceph/osd/c2-2/keyring: can't open /var/lib/ceph/osd/c2-2/keyring: (2) No such file or directory 2021-11-11T20:56:47.186 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:47.186+0000 7f91fb0d30c0 -1 created new key in keyring /var/lib/ceph/osd/c2-2/keyring 2021-11-11T20:56:47.186 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:47.186+0000 7f91fb0d30c0 -1 bluestore(/var/lib/ceph/osd/c2-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-2/block: (2) No such file or directory 2021-11-11T20:56:47.186 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:47.186+0000 7f91fb0d30c0 -1 bluestore(/var/lib/ceph/osd/c2-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-2/block: (2) No such file or directory 2021-11-11T20:56:47.187 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:47.186+0000 7f91fb0d30c0 -1 bluestore(/var/lib/ceph/osd/c2-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/c2-2/block: (2) No such file or directory 2021-11-11T20:56:47.187 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:56:47.186+0000 7f91fb0d30c0 -1 bluestore(/var/lib/ceph/osd/c2-2) _read_fsid unparsable uuid 2021-11-11T20:56:49.272 DEBUG:teuthology.orchestra.run.smithi198:> sudo chown -R ceph:ceph /var/lib/ceph/osd/c2-2 2021-11-11T20:56:49.300 INFO:tasks.ceph:Reading keys from all nodes... 2021-11-11T20:56:49.301 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:49.301 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd if=/var/lib/ceph/mgr/c2-x/keyring of=/dev/stdout 2021-11-11T20:56:49.367 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:49.367 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd if=/var/lib/ceph/osd/c2-0/keyring of=/dev/stdout 2021-11-11T20:56:49.432 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:49.433 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd if=/var/lib/ceph/osd/c2-1/keyring of=/dev/stdout 2021-11-11T20:56:49.497 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:49.498 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd if=/var/lib/ceph/osd/c2-2/keyring of=/dev/stdout 2021-11-11T20:56:49.562 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:49.562 DEBUG:teuthology.orchestra.run.smithi198:> dd if=/etc/ceph/c2.client.0.keyring of=/dev/stdout 2021-11-11T20:56:49.616 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:49.617 DEBUG:teuthology.orchestra.run.smithi198:> dd if=/etc/ceph/c2.client.1.keyring of=/dev/stdout 2021-11-11T20:56:49.671 INFO:tasks.ceph:Adding keys to all mons... 2021-11-11T20:56:49.672 DEBUG:teuthology.orchestra.run.smithi198:> sudo tee -a /etc/ceph/c2.keyring 2021-11-11T20:56:49.734 INFO:teuthology.orchestra.run.smithi198.stdout:[mgr.x] 2021-11-11T20:56:49.735 INFO:teuthology.orchestra.run.smithi198.stdout: key = AQCIg41hP8PwJRAAjernynp4ppAL6tt4jt3HeA== 2021-11-11T20:56:49.735 INFO:teuthology.orchestra.run.smithi198.stdout:[osd.0] 2021-11-11T20:56:49.735 INFO:teuthology.orchestra.run.smithi198.stdout: key = AQCKg41hyQZsLBAAc3CybrcJtNORH92whRNbMg== 2021-11-11T20:56:49.736 INFO:teuthology.orchestra.run.smithi198.stdout:[osd.1] 2021-11-11T20:56:49.736 INFO:teuthology.orchestra.run.smithi198.stdout: key = AQCMg41hHgcHOxAAT8JkARA+uFXGRw59oVhOjA== 2021-11-11T20:56:49.736 INFO:teuthology.orchestra.run.smithi198.stdout:[osd.2] 2021-11-11T20:56:49.736 INFO:teuthology.orchestra.run.smithi198.stdout: key = AQCPg41hpv4nCxAA+n9SJ5BA5akdaD/aqoAaLw== 2021-11-11T20:56:49.736 INFO:teuthology.orchestra.run.smithi198.stdout:[client.0] 2021-11-11T20:56:49.737 INFO:teuthology.orchestra.run.smithi198.stdout: key = AQCIg41hbh1IKBAAwYrWslo5R1eHOHNZr2b38g== 2021-11-11T20:56:49.737 INFO:teuthology.orchestra.run.smithi198.stdout:[client.1] 2021-11-11T20:56:49.737 INFO:teuthology.orchestra.run.smithi198.stdout: key = AQCIg41hSHSRLRAAQJAxmACUISxFRP2rD2qEPA== 2021-11-11T20:56:49.738 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2021-11-11T20:56:49.826 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2021-11-11T20:56:49.872 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2021-11-11T20:56:49.917 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2021-11-11T20:56:49.963 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2021-11-11T20:56:50.012 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/c2.keyring --name=client.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2021-11-11T20:56:50.060 INFO:tasks.ceph:Running mkfs on mon nodes... 2021-11-11T20:56:50.061 DEBUG:teuthology.orchestra.run.smithi198:> sudo mkdir -p /var/lib/ceph/mon/c2-a 2021-11-11T20:56:50.084 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster c2 --mkfs -i a --monmap /home/ubuntu/cephtest/c2.monmap --keyring /etc/ceph/c2.keyring 2021-11-11T20:56:50.174 DEBUG:teuthology.orchestra.run.smithi198:> sudo chown -R ceph:ceph /var/lib/ceph/mon/c2-a 2021-11-11T20:56:50.199 DEBUG:teuthology.orchestra.run.smithi198:> rm -- /home/ubuntu/cephtest/c2.monmap 2021-11-11T20:56:50.252 INFO:tasks.ceph:Starting mon daemons in cluster c2... 2021-11-11T20:56:50.253 INFO:tasks.ceph.c2.mon.a:Restarting daemon 2021-11-11T20:56:50.253 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster c2 -i a 2021-11-11T20:56:50.295 INFO:tasks.ceph.c2.mon.a:Started 2021-11-11T20:56:50.295 INFO:tasks.ceph:Starting mgr daemons in cluster c2... 2021-11-11T20:56:50.295 INFO:tasks.ceph.c2.mgr.x:Restarting daemon 2021-11-11T20:56:50.296 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster c2 -i x 2021-11-11T20:56:50.298 INFO:tasks.ceph.c2.mgr.x:Started 2021-11-11T20:56:50.299 INFO:tasks.ceph:Setting crush tunables to default 2021-11-11T20:56:50.299 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph --cluster c2 osd crush tunables default 2021-11-11T20:56:50.505 INFO:teuthology.orchestra.run.smithi198.stderr:adjusted tunables profile to default 2021-11-11T20:56:50.516 INFO:tasks.ceph:Starting osd daemons in cluster c2... 2021-11-11T20:56:50.516 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:50.517 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd if=/var/lib/ceph/osd/c2-0/fsid of=/dev/stdout 2021-11-11T20:56:50.540 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:50.541 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd if=/var/lib/ceph/osd/c2-1/fsid of=/dev/stdout 2021-11-11T20:56:50.604 DEBUG:teuthology.orchestra.run.smithi198:> set -ex 2021-11-11T20:56:50.604 DEBUG:teuthology.orchestra.run.smithi198:> sudo dd if=/var/lib/ceph/osd/c2-2/fsid of=/dev/stdout 2021-11-11T20:56:50.668 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph --cluster c2 osd new e34c40e0-c931-4484-add6-be39e4234902 0 2021-11-11T20:56:50.917 INFO:teuthology.orchestra.run.smithi198.stdout:0 2021-11-11T20:56:50.929 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph --cluster c2 osd new 10b35448-c4b5-421b-b760-d59600c54131 1 2021-11-11T20:56:51.138 INFO:teuthology.orchestra.run.smithi198.stdout:1 2021-11-11T20:56:51.153 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph --cluster c2 osd new 88dfbddb-71cc-40a3-87aa-6b5a2c0697ae 2 2021-11-11T20:56:51.360 INFO:teuthology.orchestra.run.smithi198.stdout:2 2021-11-11T20:56:51.373 INFO:tasks.ceph.c2.osd.0:Restarting daemon 2021-11-11T20:56:51.374 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 0 2021-11-11T20:56:51.376 INFO:tasks.ceph.c2.osd.0:Started 2021-11-11T20:56:51.376 INFO:tasks.ceph.c2.osd.1:Restarting daemon 2021-11-11T20:56:51.377 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 1 2021-11-11T20:56:51.379 INFO:tasks.ceph.c2.osd.1:Started 2021-11-11T20:56:51.379 INFO:tasks.ceph.c2.osd.2:Restarting daemon 2021-11-11T20:56:51.379 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster c2 -i 2 2021-11-11T20:56:51.382 INFO:tasks.ceph.c2.osd.2:Started 2021-11-11T20:56:51.382 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd dump --format=json 2021-11-11T20:56:51.592 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:56:51.592 INFO:teuthology.orchestra.run.smithi198.stdout:{"epoch":5,"fsid":"e2c5170a-0bbc-4bab-972e-5a438845bdd2","created":"2021-11-11T20:56:50.411394+0000","modified":"2021-11-11T20:56:51.355498+0000","last_up_change":"0.000000","last_in_change":"2021-11-11T20:56:51.355498+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[],"osds":[{"osd":0,"uuid":"e34c40e0-c931-4484-add6-be39e4234902","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"10b35448-c4b5-421b-b760-d59600c54131","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"88dfbddb-71cc-40a3-87aa-6b5a2c0697ae","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2021-11-11T20:56:51.604 INFO:tasks.ceph.ceph_manager.c2:[] 2021-11-11T20:56:51.605 INFO:tasks.ceph:Waiting for OSDs to come up 2021-11-11T20:56:51.908 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c2 osd dump --format=json 2021-11-11T20:56:51.971 INFO:tasks.ceph.c2.osd.1.smithi198.stderr:2021-11-11T20:56:51.971+0000 7fcc61c3f0c0 -1 Falling back to public interface 2021-11-11T20:56:51.992 INFO:tasks.ceph.c2.osd.0.smithi198.stderr:2021-11-11T20:56:51.992+0000 7f6e24da60c0 -1 Falling back to public interface 2021-11-11T20:56:51.996 INFO:tasks.ceph.c2.osd.2.smithi198.stderr:2021-11-11T20:56:51.996+0000 7f5aeee200c0 -1 Falling back to public interface 2021-11-11T20:56:52.089 INFO:teuthology.misc.health.smithi198.stdout: 2021-11-11T20:56:52.090 INFO:teuthology.misc.health.smithi198.stdout:{"epoch":5,"fsid":"e2c5170a-0bbc-4bab-972e-5a438845bdd2","created":"2021-11-11T20:56:50.411394+0000","modified":"2021-11-11T20:56:51.355498+0000","last_up_change":"0.000000","last_in_change":"2021-11-11T20:56:51.355498+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[],"osds":[{"osd":0,"uuid":"e34c40e0-c931-4484-add6-be39e4234902","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"10b35448-c4b5-421b-b760-d59600c54131","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"88dfbddb-71cc-40a3-87aa-6b5a2c0697ae","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2021-11-11T20:56:52.101 DEBUG:teuthology.misc:0 of 3 OSDs are up 2021-11-11T20:56:54.686 INFO:tasks.ceph.c2.osd.1.smithi198.stderr:2021-11-11T20:56:54.686+0000 7fcc61c3f0c0 -1 osd.1 0 log_to_monitors {default=true} 2021-11-11T20:56:54.702 INFO:tasks.ceph.c2.osd.0.smithi198.stderr:2021-11-11T20:56:54.703+0000 7f6e24da60c0 -1 osd.0 0 log_to_monitors {default=true} 2021-11-11T20:56:54.730 INFO:tasks.ceph.c2.osd.2.smithi198.stderr:2021-11-11T20:56:54.730+0000 7f5aeee200c0 -1 osd.2 0 log_to_monitors {default=true} 2021-11-11T20:56:56.883 INFO:tasks.ceph.c2.osd.2.smithi198.stderr:2021-11-11T20:56:56.883+0000 7f5ae57de700 -1 osd.2 0 waiting for initial osdmap 2021-11-11T20:56:56.885 INFO:tasks.ceph.c2.osd.2.smithi198.stderr:2021-11-11T20:56:56.885+0000 7f5ade171700 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2021-11-11T20:56:57.053 INFO:tasks.ceph.c2.osd.0.smithi198.stderr:2021-11-11T20:56:57.053+0000 7f6e1b764700 -1 osd.0 0 waiting for initial osdmap 2021-11-11T20:56:57.056 INFO:tasks.ceph.c2.osd.0.smithi198.stderr:2021-11-11T20:56:57.056+0000 7f6e140f7700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2021-11-11T20:56:57.341 INFO:tasks.ceph.c2.osd.1.smithi198.stderr:2021-11-11T20:56:57.341+0000 7fcc585fd700 -1 osd.1 0 waiting for initial osdmap 2021-11-11T20:56:57.342 INFO:tasks.ceph.c2.osd.1.smithi198.stderr:2021-11-11T20:56:57.343+0000 7fcc50f90700 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2021-11-11T20:56:58.404 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster c2 osd dump --format=json 2021-11-11T20:56:58.686 INFO:teuthology.misc.health.smithi198.stdout: 2021-11-11T20:56:58.687 INFO:teuthology.misc.health.smithi198.stdout:{"epoch":8,"fsid":"e2c5170a-0bbc-4bab-972e-5a438845bdd2","created":"2021-11-11T20:56:50.411394+0000","modified":"2021-11-11T20:56:57.882438+0000","last_up_change":"2021-11-11T20:56:57.882438+0000","last_in_change":"2021-11-11T20:56:51.355498+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[],"osds":[{"osd":0,"uuid":"e34c40e0-c931-4484-add6-be39e4234902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6808","nonce":32199},{"type":"v1","addr":"172.21.15.198:6809","nonce":32199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6810","nonce":32199},{"type":"v1","addr":"172.21.15.198:6811","nonce":32199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6814","nonce":32199},{"type":"v1","addr":"172.21.15.198:6815","nonce":32199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6812","nonce":32199},{"type":"v1","addr":"172.21.15.198:6813","nonce":32199}]},"public_addr":"172.21.15.198:6809/32199","cluster_addr":"172.21.15.198:6811/32199","heartbeat_back_addr":"172.21.15.198:6815/32199","heartbeat_front_addr":"172.21.15.198:6813/32199","state":["exists","up"]},{"osd":1,"uuid":"10b35448-c4b5-421b-b760-d59600c54131","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6800","nonce":32184},{"type":"v1","addr":"172.21.15.198:6801","nonce":32184}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6802","nonce":32184},{"type":"v1","addr":"172.21.15.198:6803","nonce":32184}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6806","nonce":32184},{"type":"v1","addr":"172.21.15.198:6807","nonce":32184}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6804","nonce":32184},{"type":"v1","addr":"172.21.15.198:6805","nonce":32184}]},"public_addr":"172.21.15.198:6801/32184","cluster_addr":"172.21.15.198:6803/32184","heartbeat_back_addr":"172.21.15.198:6807/32184","heartbeat_front_addr":"172.21.15.198:6805/32184","state":["exists","up"]},{"osd":2,"uuid":"88dfbddb-71cc-40a3-87aa-6b5a2c0697ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6816","nonce":32200},{"type":"v1","addr":"172.21.15.198:6817","nonce":32200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6818","nonce":32200},{"type":"v1","addr":"172.21.15.198:6819","nonce":32200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6822","nonce":32200},{"type":"v1","addr":"172.21.15.198:6823","nonce":32200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6820","nonce":32200},{"type":"v1","addr":"172.21.15.198:6821","nonce":32200}]},"public_addr":"172.21.15.198:6817/32200","cluster_addr":"172.21.15.198:6819/32200","heartbeat_back_addr":"172.21.15.198:6823/32200","heartbeat_front_addr":"172.21.15.198:6821/32200","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2021-11-11T20:56:58.698 DEBUG:teuthology.misc:3 of 3 OSDs are up 2021-11-11T20:56:58.698 INFO:tasks.ceph:Creating RBD pool 2021-11-11T20:56:58.699 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph --cluster c2 osd pool create rbd 8 2021-11-11T20:56:59.531 INFO:teuthology.orchestra.run.smithi198.stderr:pool 'rbd' created 2021-11-11T20:56:59.542 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph --cluster c2 osd pool application enable rbd rbd --yes-i-really-mean-it 2021-11-11T20:57:00.530 INFO:teuthology.orchestra.run.smithi198.stderr:enabled application 'rbd' on pool 'rbd' 2021-11-11T20:57:00.548 INFO:tasks.ceph:Starting mds daemons in cluster c2... 2021-11-11T20:57:00.548 INFO:tasks.ceph:Waiting until c2 daemons up and pgs clean... 2021-11-11T20:57:00.549 INFO:tasks.ceph.ceph_manager.c2:waiting for mgr available 2021-11-11T20:57:00.549 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 mgr dump --format=json 2021-11-11T20:57:00.550 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2021-11-11T20:57:00.960 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:57:00.975 INFO:teuthology.orchestra.run.smithi198.stdout:{"epoch":4,"active_gid":4101,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6824","nonce":31845},{"type":"v1","addr":"172.21.15.198:6825","nonce":31845}]},"active_addr":"172.21.15.198:6825/31845","active_change":"2021-11-11T20:56:55.446885+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Registry to which we should normalize unqualified image names","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"autoscale_profile":{"name":"autoscale_profile","type":"str","level":"advanced","flags":1,"default_value":"scale-up","min":"","max":"","enum_allowed":[],"desc":"pg_autoscale profiler","long_desc":"Determines the behavior of the autoscaler algorithm `scale-up` means that it starts out with minmum pgs and scales up when there is pressure, `scale-down` means starts out with full pgs and scales down when there is pressure ","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share perf counter metrics summed across the whole cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":0,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.198:0","nonce":894734310}]},{"addrvec":[{"type":"v2","addr":"172.21.15.198:0","nonce":927705955}]},{"addrvec":[{"type":"v2","addr":"172.21.15.198:0","nonce":4239180847}]}]}} 2021-11-11T20:57:00.978 INFO:tasks.ceph.ceph_manager.c2:mgr available! 2021-11-11T20:57:00.978 INFO:tasks.ceph.ceph_manager.c2:waiting for all up 2021-11-11T20:57:00.979 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd dump --format=json 2021-11-11T20:57:01.257 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:57:01.257 INFO:teuthology.orchestra.run.smithi198.stdout:{"epoch":10,"fsid":"e2c5170a-0bbc-4bab-972e-5a438845bdd2","created":"2021-11-11T20:56:50.411394+0000","modified":"2021-11-11T20:57:00.521957+0000","last_up_change":"2021-11-11T20:56:57.882438+0000","last_in_change":"2021-11-11T20:56:51.355498+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":"rbd","create_time":"2021-11-11T20:56:58.993109+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":2,"pool_name":".mgr","create_time":"2021-11-11T20:56:59.479012+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"e34c40e0-c931-4484-add6-be39e4234902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6808","nonce":32199},{"type":"v1","addr":"172.21.15.198:6809","nonce":32199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6810","nonce":32199},{"type":"v1","addr":"172.21.15.198:6811","nonce":32199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6814","nonce":32199},{"type":"v1","addr":"172.21.15.198:6815","nonce":32199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6812","nonce":32199},{"type":"v1","addr":"172.21.15.198:6813","nonce":32199}]},"public_addr":"172.21.15.198:6809/32199","cluster_addr":"172.21.15.198:6811/32199","heartbeat_back_addr":"172.21.15.198:6815/32199","heartbeat_front_addr":"172.21.15.198:6813/32199","state":["exists","up"]},{"osd":1,"uuid":"10b35448-c4b5-421b-b760-d59600c54131","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6800","nonce":32184},{"type":"v1","addr":"172.21.15.198:6801","nonce":32184}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6802","nonce":32184},{"type":"v1","addr":"172.21.15.198:6803","nonce":32184}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6806","nonce":32184},{"type":"v1","addr":"172.21.15.198:6807","nonce":32184}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6804","nonce":32184},{"type":"v1","addr":"172.21.15.198:6805","nonce":32184}]},"public_addr":"172.21.15.198:6801/32184","cluster_addr":"172.21.15.198:6803/32184","heartbeat_back_addr":"172.21.15.198:6807/32184","heartbeat_front_addr":"172.21.15.198:6805/32184","state":["exists","up"]},{"osd":2,"uuid":"88dfbddb-71cc-40a3-87aa-6b5a2c0697ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6816","nonce":32200},{"type":"v1","addr":"172.21.15.198:6817","nonce":32200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6818","nonce":32200},{"type":"v1","addr":"172.21.15.198:6819","nonce":32200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6822","nonce":32200},{"type":"v1","addr":"172.21.15.198:6823","nonce":32200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6820","nonce":32200},{"type":"v1","addr":"172.21.15.198:6821","nonce":32200}]},"public_addr":"172.21.15.198:6817/32200","cluster_addr":"172.21.15.198:6819/32200","heartbeat_back_addr":"172.21.15.198:6823/32200","heartbeat_front_addr":"172.21.15.198:6821/32200","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:55.701486+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:55.734110+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:55.738272+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2021-11-11T20:57:01.270 INFO:tasks.ceph.ceph_manager.c2:all up! 2021-11-11T20:57:01.270 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd dump --format=json 2021-11-11T20:57:01.550 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:57:01.551 INFO:teuthology.orchestra.run.smithi198.stdout:{"epoch":11,"fsid":"e2c5170a-0bbc-4bab-972e-5a438845bdd2","created":"2021-11-11T20:56:50.411394+0000","modified":"2021-11-11T20:57:01.531404+0000","last_up_change":"2021-11-11T20:56:57.882438+0000","last_in_change":"2021-11-11T20:56:51.355498+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":"rbd","create_time":"2021-11-11T20:56:58.993109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":2,"pool_name":".mgr","create_time":"2021-11-11T20:56:59.479012+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"e34c40e0-c931-4484-add6-be39e4234902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6808","nonce":32199},{"type":"v1","addr":"172.21.15.198:6809","nonce":32199}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6810","nonce":32199},{"type":"v1","addr":"172.21.15.198:6811","nonce":32199}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6814","nonce":32199},{"type":"v1","addr":"172.21.15.198:6815","nonce":32199}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6812","nonce":32199},{"type":"v1","addr":"172.21.15.198:6813","nonce":32199}]},"public_addr":"172.21.15.198:6809/32199","cluster_addr":"172.21.15.198:6811/32199","heartbeat_back_addr":"172.21.15.198:6815/32199","heartbeat_front_addr":"172.21.15.198:6813/32199","state":["exists","up"]},{"osd":1,"uuid":"10b35448-c4b5-421b-b760-d59600c54131","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6800","nonce":32184},{"type":"v1","addr":"172.21.15.198:6801","nonce":32184}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6802","nonce":32184},{"type":"v1","addr":"172.21.15.198:6803","nonce":32184}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6806","nonce":32184},{"type":"v1","addr":"172.21.15.198:6807","nonce":32184}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6804","nonce":32184},{"type":"v1","addr":"172.21.15.198:6805","nonce":32184}]},"public_addr":"172.21.15.198:6801/32184","cluster_addr":"172.21.15.198:6803/32184","heartbeat_back_addr":"172.21.15.198:6807/32184","heartbeat_front_addr":"172.21.15.198:6805/32184","state":["exists","up"]},{"osd":2,"uuid":"88dfbddb-71cc-40a3-87aa-6b5a2c0697ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6816","nonce":32200},{"type":"v1","addr":"172.21.15.198:6817","nonce":32200}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6818","nonce":32200},{"type":"v1","addr":"172.21.15.198:6819","nonce":32200}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6822","nonce":32200},{"type":"v1","addr":"172.21.15.198:6823","nonce":32200}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.198:6820","nonce":32200},{"type":"v1","addr":"172.21.15.198:6821","nonce":32200}]},"public_addr":"172.21.15.198:6817/32200","cluster_addr":"172.21.15.198:6819/32200","heartbeat_back_addr":"172.21.15.198:6823/32200","heartbeat_front_addr":"172.21.15.198:6821/32200","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:55.701486+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:55.734110+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-11-11T20:56:55.738272+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2021-11-11T20:57:01.564 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.0 flush_pg_stats 2021-11-11T20:57:01.708 INFO:teuthology.orchestra.run.smithi198.stdout:34359738372 2021-11-11T20:57:01.709 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.1 flush_pg_stats 2021-11-11T20:57:01.855 INFO:teuthology.orchestra.run.smithi198.stdout:34359738372 2021-11-11T20:57:01.855 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 tell osd.2 flush_pg_stats 2021-11-11T20:57:01.999 INFO:teuthology.orchestra.run.smithi198.stdout:34359738372 2021-11-11T20:57:01.999 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.0 2021-11-11T20:57:02.277 INFO:teuthology.orchestra.run.smithi198.stdout:34359738371 2021-11-11T20:57:02.290 INFO:tasks.ceph.ceph_manager.c2:need seq 34359738372 got 34359738371 for osd.0 2021-11-11T20:57:03.291 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.0 2021-11-11T20:57:03.585 INFO:teuthology.orchestra.run.smithi198.stdout:34359738372 2021-11-11T20:57:03.598 INFO:tasks.ceph.ceph_manager.c2:need seq 34359738372 got 34359738372 for osd.0 2021-11-11T20:57:03.599 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.1 2021-11-11T20:57:03.880 INFO:teuthology.orchestra.run.smithi198.stdout:34359738372 2021-11-11T20:57:03.893 INFO:tasks.ceph.ceph_manager.c2:need seq 34359738372 got 34359738372 for osd.1 2021-11-11T20:57:03.893 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 osd last-stat-seq osd.2 2021-11-11T20:57:04.174 INFO:teuthology.orchestra.run.smithi198.stdout:34359738372 2021-11-11T20:57:04.187 INFO:tasks.ceph.ceph_manager.c2:need seq 34359738372 got 34359738372 for osd.2 2021-11-11T20:57:04.188 INFO:tasks.ceph.ceph_manager.c2:waiting for clean 2021-11-11T20:57:04.188 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 pg dump --format=json 2021-11-11T20:57:04.462 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:57:04.464 INFO:teuthology.orchestra.run.smithi198.stderr:dumped all 2021-11-11T20:57:04.475 INFO:teuthology.orchestra.run.smithi198.stdout:{"pg_ready":true,"pg_map":{"version":11,"stamp":"2021-11-11T20:57:03.451858+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":11,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":314572800,"kb_used":25124,"kb_used_data":1428,"kb_used_omap":0,"kb_used_meta":23168,"kb_avail":314547676,"statfs":{"total":322122547200,"available":322096820224,"internally_reserved":0,"allocated":1462272,"data_stored":1250617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":23724032},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":9,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.538193+0000","last_change":"2021-11-11T20:57:00.536583+0000","last_active":"2021-11-11T20:57:00.538193+0000","last_peered":"2021-11-11T20:57:00.538193+0000","last_clean":"2021-11-11T20:57:00.538193+0000","last_became_active":"2021-11-11T20:57:00.536214+0000","last_became_peered":"2021-11-11T20:57:00.536214+0000","last_unstale":"2021-11-11T20:57:00.538193+0000","last_undegraded":"2021-11-11T20:57:00.538193+0000","last_fullsized":"2021-11-11T20:57:00.538193+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T05:29:40.026973+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":9,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.543081+0000","last_change":"2021-11-11T20:57:00.542035+0000","last_active":"2021-11-11T20:57:00.543081+0000","last_peered":"2021-11-11T20:57:00.543081+0000","last_clean":"2021-11-11T20:57:00.543081+0000","last_became_active":"2021-11-11T20:57:00.541689+0000","last_became_peered":"2021-11-11T20:57:00.541689+0000","last_unstale":"2021-11-11T20:57:00.543081+0000","last_undegraded":"2021-11-11T20:57:00.543081+0000","last_fullsized":"2021-11-11T20:57:00.543081+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-12T21:29:38.629669+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":8,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.537359+0000","last_change":"2021-11-11T20:57:00.537359+0000","last_active":"2021-11-11T20:57:00.537359+0000","last_peered":"2021-11-11T20:57:00.537359+0000","last_clean":"2021-11-11T20:57:00.537359+0000","last_became_active":"2021-11-11T20:57:00.537013+0000","last_became_peered":"2021-11-11T20:57:00.537013+0000","last_unstale":"2021-11-11T20:57:00.537359+0000","last_undegraded":"2021-11-11T20:57:00.537359+0000","last_fullsized":"2021-11-11T20:57:00.537359+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T00:47:38.826874+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":9,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.540673+0000","last_change":"2021-11-11T20:57:00.539634+0000","last_active":"2021-11-11T20:57:00.540673+0000","last_peered":"2021-11-11T20:57:00.540673+0000","last_clean":"2021-11-11T20:57:00.540673+0000","last_became_active":"2021-11-11T20:57:00.539139+0000","last_became_peered":"2021-11-11T20:57:00.539139+0000","last_unstale":"2021-11-11T20:57:00.540673+0000","last_undegraded":"2021-11-11T20:57:00.540673+0000","last_fullsized":"2021-11-11T20:57:00.540673+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T08:26:57.714360+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":8,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.535452+0000","last_change":"2021-11-11T20:57:00.535452+0000","last_active":"2021-11-11T20:57:00.535452+0000","last_peered":"2021-11-11T20:57:00.535452+0000","last_clean":"2021-11-11T20:57:00.535452+0000","last_became_active":"2021-11-11T20:57:00.535107+0000","last_became_peered":"2021-11-11T20:57:00.535107+0000","last_unstale":"2021-11-11T20:57:00.535452+0000","last_undegraded":"2021-11-11T20:57:00.535452+0000","last_fullsized":"2021-11-11T20:57:00.535452+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T00:43:49.321037+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"10'97","reported_seq":135,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.979363+0000","last_change":"2021-11-11T20:57:00.536850+0000","last_active":"2021-11-11T20:57:00.979363+0000","last_peered":"2021-11-11T20:57:00.979363+0000","last_clean":"2021-11-11T20:57:00.979363+0000","last_became_active":"2021-11-11T20:57:00.535796+0000","last_became_peered":"2021-11-11T20:57:00.535796+0000","last_unstale":"2021-11-11T20:57:00.979363+0000","last_undegraded":"2021-11-11T20:57:00.979363+0000","last_fullsized":"2021-11-11T20:57:00.979363+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":97,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T08:38:39.146806+0000","stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":9,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.540689+0000","last_change":"2021-11-11T20:57:00.539862+0000","last_active":"2021-11-11T20:57:00.540689+0000","last_peered":"2021-11-11T20:57:00.540689+0000","last_clean":"2021-11-11T20:57:00.540689+0000","last_became_active":"2021-11-11T20:57:00.539550+0000","last_became_peered":"2021-11-11T20:57:00.539550+0000","last_unstale":"2021-11-11T20:57:00.540689+0000","last_undegraded":"2021-11-11T20:57:00.540689+0000","last_fullsized":"2021-11-11T20:57:00.540689+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T05:47:21.806569+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":9,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.540759+0000","last_change":"2021-11-11T20:57:00.539701+0000","last_active":"2021-11-11T20:57:00.540759+0000","last_peered":"2021-11-11T20:57:00.540759+0000","last_clean":"2021-11-11T20:57:00.540759+0000","last_became_active":"2021-11-11T20:57:00.539223+0000","last_became_peered":"2021-11-11T20:57:00.539223+0000","last_unstale":"2021-11-11T20:57:00.540759+0000","last_undegraded":"2021-11-11T20:57:00.540759+0000","last_fullsized":"2021-11-11T20:57:00.540759+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T05:35:16.527099+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":8,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.537087+0000","last_change":"2021-11-11T20:57:00.537087+0000","last_active":"2021-11-11T20:57:00.537087+0000","last_peered":"2021-11-11T20:57:00.537087+0000","last_clean":"2021-11-11T20:57:00.537087+0000","last_became_active":"2021-11-11T20:57:00.536821+0000","last_became_peered":"2021-11-11T20:57:00.536821+0000","last_unstale":"2021-11-11T20:57:00.537087+0000","last_undegraded":"2021-11-11T20:57:00.537087+0000","last_fullsized":"2021-11-11T20:57:00.537087+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-12T21:38:10.947933+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8984,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":7808,"kb_avail":104848616,"statfs":{"total":107374182400,"available":107364982784,"internally_reserved":0,"allocated":679936,"data_stored":611669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7995392},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53300000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7584,"kb_used_data":88,"kb_used_omap":0,"kb_used_meta":7488,"kb_avail":104850016,"statfs":{"total":107374182400,"available":107366416384,"internally_reserved":0,"allocated":90112,"data_stored":22388,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7667712},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.621}]}]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8556,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":7872,"kb_avail":104849044,"statfs":{"total":107374182400,"available":107365421056,"internally_reserved":0,"allocated":692224,"data_stored":616560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8060928},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76200000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70899999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2021-11-11T20:57:04.476 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 pg dump --format=json 2021-11-11T20:57:04.751 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:57:04.753 INFO:teuthology.orchestra.run.smithi198.stderr:dumped all 2021-11-11T20:57:04.764 INFO:teuthology.orchestra.run.smithi198.stdout:{"pg_ready":true,"pg_map":{"version":11,"stamp":"2021-11-11T20:57:03.451858+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":11,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":314572800,"kb_used":25124,"kb_used_data":1428,"kb_used_omap":0,"kb_used_meta":23168,"kb_avail":314547676,"statfs":{"total":322122547200,"available":322096820224,"internally_reserved":0,"allocated":1462272,"data_stored":1250617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":23724032},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":9,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.538193+0000","last_change":"2021-11-11T20:57:00.536583+0000","last_active":"2021-11-11T20:57:00.538193+0000","last_peered":"2021-11-11T20:57:00.538193+0000","last_clean":"2021-11-11T20:57:00.538193+0000","last_became_active":"2021-11-11T20:57:00.536214+0000","last_became_peered":"2021-11-11T20:57:00.536214+0000","last_unstale":"2021-11-11T20:57:00.538193+0000","last_undegraded":"2021-11-11T20:57:00.538193+0000","last_fullsized":"2021-11-11T20:57:00.538193+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T05:29:40.026973+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":9,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.543081+0000","last_change":"2021-11-11T20:57:00.542035+0000","last_active":"2021-11-11T20:57:00.543081+0000","last_peered":"2021-11-11T20:57:00.543081+0000","last_clean":"2021-11-11T20:57:00.543081+0000","last_became_active":"2021-11-11T20:57:00.541689+0000","last_became_peered":"2021-11-11T20:57:00.541689+0000","last_unstale":"2021-11-11T20:57:00.543081+0000","last_undegraded":"2021-11-11T20:57:00.543081+0000","last_fullsized":"2021-11-11T20:57:00.543081+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-12T21:29:38.629669+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":8,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.537359+0000","last_change":"2021-11-11T20:57:00.537359+0000","last_active":"2021-11-11T20:57:00.537359+0000","last_peered":"2021-11-11T20:57:00.537359+0000","last_clean":"2021-11-11T20:57:00.537359+0000","last_became_active":"2021-11-11T20:57:00.537013+0000","last_became_peered":"2021-11-11T20:57:00.537013+0000","last_unstale":"2021-11-11T20:57:00.537359+0000","last_undegraded":"2021-11-11T20:57:00.537359+0000","last_fullsized":"2021-11-11T20:57:00.537359+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T00:47:38.826874+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":9,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.540673+0000","last_change":"2021-11-11T20:57:00.539634+0000","last_active":"2021-11-11T20:57:00.540673+0000","last_peered":"2021-11-11T20:57:00.540673+0000","last_clean":"2021-11-11T20:57:00.540673+0000","last_became_active":"2021-11-11T20:57:00.539139+0000","last_became_peered":"2021-11-11T20:57:00.539139+0000","last_unstale":"2021-11-11T20:57:00.540673+0000","last_undegraded":"2021-11-11T20:57:00.540673+0000","last_fullsized":"2021-11-11T20:57:00.540673+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T08:26:57.714360+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":8,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.535452+0000","last_change":"2021-11-11T20:57:00.535452+0000","last_active":"2021-11-11T20:57:00.535452+0000","last_peered":"2021-11-11T20:57:00.535452+0000","last_clean":"2021-11-11T20:57:00.535452+0000","last_became_active":"2021-11-11T20:57:00.535107+0000","last_became_peered":"2021-11-11T20:57:00.535107+0000","last_unstale":"2021-11-11T20:57:00.535452+0000","last_undegraded":"2021-11-11T20:57:00.535452+0000","last_fullsized":"2021-11-11T20:57:00.535452+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T00:43:49.321037+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"10'97","reported_seq":135,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.979363+0000","last_change":"2021-11-11T20:57:00.536850+0000","last_active":"2021-11-11T20:57:00.979363+0000","last_peered":"2021-11-11T20:57:00.979363+0000","last_clean":"2021-11-11T20:57:00.979363+0000","last_became_active":"2021-11-11T20:57:00.535796+0000","last_became_peered":"2021-11-11T20:57:00.535796+0000","last_unstale":"2021-11-11T20:57:00.979363+0000","last_undegraded":"2021-11-11T20:57:00.979363+0000","last_fullsized":"2021-11-11T20:57:00.979363+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":97,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T08:38:39.146806+0000","stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":9,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.540689+0000","last_change":"2021-11-11T20:57:00.539862+0000","last_active":"2021-11-11T20:57:00.540689+0000","last_peered":"2021-11-11T20:57:00.540689+0000","last_clean":"2021-11-11T20:57:00.540689+0000","last_became_active":"2021-11-11T20:57:00.539550+0000","last_became_peered":"2021-11-11T20:57:00.539550+0000","last_unstale":"2021-11-11T20:57:00.540689+0000","last_undegraded":"2021-11-11T20:57:00.540689+0000","last_fullsized":"2021-11-11T20:57:00.540689+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T05:47:21.806569+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":9,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.540759+0000","last_change":"2021-11-11T20:57:00.539701+0000","last_active":"2021-11-11T20:57:00.540759+0000","last_peered":"2021-11-11T20:57:00.540759+0000","last_clean":"2021-11-11T20:57:00.540759+0000","last_became_active":"2021-11-11T20:57:00.539223+0000","last_became_peered":"2021-11-11T20:57:00.539223+0000","last_unstale":"2021-11-11T20:57:00.540759+0000","last_undegraded":"2021-11-11T20:57:00.540759+0000","last_fullsized":"2021-11-11T20:57:00.540759+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-13T05:35:16.527099+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":8,"reported_epoch":10,"state":"active+clean","last_fresh":"2021-11-11T20:57:00.537087+0000","last_change":"2021-11-11T20:57:00.537087+0000","last_active":"2021-11-11T20:57:00.537087+0000","last_peered":"2021-11-11T20:57:00.537087+0000","last_clean":"2021-11-11T20:57:00.537087+0000","last_became_active":"2021-11-11T20:57:00.536821+0000","last_became_peered":"2021-11-11T20:57:00.536821+0000","last_unstale":"2021-11-11T20:57:00.537087+0000","last_undegraded":"2021-11-11T20:57:00.537087+0000","last_fullsized":"2021-11-11T20:57:00.537087+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2021-11-11T20:56:59.519937+0000","last_clean_scrub_stamp":"2021-11-11T20:56:59.519937+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2021-11-12T21:38:10.947933+0000","stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8984,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":7808,"kb_avail":104848616,"statfs":{"total":107374182400,"available":107364982784,"internally_reserved":0,"allocated":679936,"data_stored":611669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7995392},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53300000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":7584,"kb_used_data":88,"kb_used_omap":0,"kb_used_meta":7488,"kb_avail":104850016,"statfs":{"total":107374182400,"available":107366416384,"internally_reserved":0,"allocated":90112,"data_stored":22388,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":7667712},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.621}]}]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8556,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":7872,"kb_avail":104849044,"statfs":{"total":107374182400,"available":107365421056,"internally_reserved":0,"allocated":692224,"data_stored":616560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8060928},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76200000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70899999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2021-11-11T20:57:04.765 INFO:tasks.ceph.ceph_manager.c2:clean! 2021-11-11T20:57:04.766 INFO:tasks.ceph:Waiting until ceph cluster c2 is healthy... 2021-11-11T20:57:04.766 INFO:tasks.ceph.ceph_manager.c2:wait_until_healthy 2021-11-11T20:57:04.766 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster c2 health --format=json 2021-11-11T20:57:05.081 INFO:teuthology.orchestra.run.smithi198.stdout: 2021-11-11T20:57:05.081 INFO:teuthology.orchestra.run.smithi198.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2021-11-11T20:57:05.094 INFO:tasks.ceph.ceph_manager.c2:wait_until_healthy done 2021-11-11T20:57:05.095 INFO:teuthology.run_tasks:Running task rgw... 2021-11-11T20:57:05.111 DEBUG:tasks.rgw:config is {'c1.client.0': {'port': 8000}, 'c1.client.1': {'port': 8001}, 'c2.client.0': {'port': 8000}, 'c2.client.1': {'port': 8001}} 2021-11-11T20:57:05.111 DEBUG:tasks.rgw:client list is dict_keys(['c1.client.0', 'c1.client.1', 'c2.client.0', 'c2.client.1']) 2021-11-11T20:57:05.112 INFO:tasks.rgw:Creating data pools 2021-11-11T20:57:05.112 DEBUG:tasks.rgw:Obtaining remote for client c1.client.0 2021-11-11T20:57:05.113 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c1 2021-11-11T20:57:05.507 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'default.rgw.buckets.data' created 2021-11-11T20:57:05.517 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c1 2021-11-11T20:57:06.523 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2021-11-11T20:57:06.542 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c1 2021-11-11T20:57:07.521 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'default.rgw.buckets.index' created 2021-11-11T20:57:07.533 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c1 2021-11-11T20:57:08.525 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2021-11-11T20:57:08.540 DEBUG:tasks.rgw:Obtaining remote for client c1.client.1 2021-11-11T20:57:08.541 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c1 2021-11-11T20:57:08.835 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'default.rgw.buckets.data' already exists 2021-11-11T20:57:08.845 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c1 2021-11-11T20:57:09.513 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2021-11-11T20:57:09.527 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c1 2021-11-11T20:57:09.814 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'default.rgw.buckets.index' already exists 2021-11-11T20:57:09.824 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c1 2021-11-11T20:57:10.519 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2021-11-11T20:57:10.531 DEBUG:tasks.rgw:Obtaining remote for client c2.client.0 2021-11-11T20:57:10.532 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c2 2021-11-11T20:57:11.561 INFO:teuthology.orchestra.run.smithi198.stderr:pool 'default.rgw.buckets.data' created 2021-11-11T20:57:11.579 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c2 2021-11-11T20:57:12.571 INFO:teuthology.orchestra.run.smithi198.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2021-11-11T20:57:12.593 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c2 2021-11-11T20:57:13.565 INFO:teuthology.orchestra.run.smithi198.stderr:pool 'default.rgw.buckets.index' created 2021-11-11T20:57:13.583 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c2 2021-11-11T20:57:14.585 INFO:teuthology.orchestra.run.smithi198.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2021-11-11T20:57:14.598 DEBUG:tasks.rgw:Obtaining remote for client c2.client.1 2021-11-11T20:57:14.599 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool create default.rgw.buckets.data 64 64 --cluster c2 2021-11-11T20:57:14.889 INFO:teuthology.orchestra.run.smithi198.stderr:pool 'default.rgw.buckets.data' already exists 2021-11-11T20:57:14.899 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool application enable default.rgw.buckets.data rgw --cluster c2 2021-11-11T20:57:15.575 INFO:teuthology.orchestra.run.smithi198.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.data' 2021-11-11T20:57:15.586 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool create default.rgw.buckets.index 64 64 --cluster c2 2021-11-11T20:57:15.875 INFO:teuthology.orchestra.run.smithi198.stderr:pool 'default.rgw.buckets.index' already exists 2021-11-11T20:57:15.885 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool application enable default.rgw.buckets.index rgw --cluster c2 2021-11-11T20:57:16.576 INFO:teuthology.orchestra.run.smithi198.stderr:enabled application 'rgw' on pool 'default.rgw.buckets.index' 2021-11-11T20:57:16.587 DEBUG:tasks.rgw:Pools created 2021-11-11T20:57:16.587 INFO:tasks.rgw:Configuring compression type = random 2021-11-11T20:57:16.588 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'list'] 2021-11-11T20:57:16.588 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'list'] 2021-11-11T20:57:16.589 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user list 2021-11-11T20:57:16.663 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:57:16.664 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:57:16.669 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:16.670+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.756 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.756+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.756 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.756+0000 7fec80ff8380 20 realm 2021-11-11T20:57:17.756 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.756+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.757 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.758+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.757 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.758+0000 7fec80ff8380 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2021-11-11T20:57:17.757 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.758+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.761 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.762+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.761 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.762+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.762 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.762+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.762 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.762+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.763 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.764+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.763 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.764+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.764 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.764+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.764 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.764+0000 7fec80ff8380 10 Creating default zonegroup 2021-11-11T20:57:17.764 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.764+0000 7fec80ff8380 20 get_system_obj_state: rctx=0x7ffe8489b710 obj=:.pools.avail state=0x557a652d5eb0 s->prefetch_data=0 2021-11-11T20:57:17.765 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.765+0000 7fec80ff8380 10 couldn't find old data placement pools config, setting up new ones for the zone 2021-11-11T20:57:17.780 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.780+0000 7fec80ff8380 10 WARNING: store->list_zones() returned r=-2 2021-11-11T20:57:17.780 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.780+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.780 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.781+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.783 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.784+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.784 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.784+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.784 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.784+0000 7fec80ff8380 10 could not read realm id: (2) No such file or directory 2021-11-11T20:57:17.784 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.784+0000 7fec80ff8380 10 WARNING: failed to set zone as default, r=-22 2021-11-11T20:57:17.784 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.784+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.785 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.784+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.787 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.788+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.788 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.788+0000 7fec80ff8380 20 rados_obj.operate() r=0 bl.length=873 2021-11-11T20:57:17.788 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.788+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.789 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.789+0000 7fec80ff8380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:17.789 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.789+0000 7fec80ff8380 10 Using default name default 2021-11-11T20:57:17.790 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.789+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.790 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.789+0000 7fec80ff8380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:17.790 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.789+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.790 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.790+0000 7fec80ff8380 20 rados_obj.operate() r=0 bl.length=873 2021-11-11T20:57:17.791 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.790+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.791 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.790+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.791 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.790+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.792 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.791+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.792 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.791+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.792 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.791+0000 7fec80ff8380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:17.801 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.802+0000 7fec80ff8380 20 RGWRados::pool_iterate: got zone_info.e42aa540-4a3f-4162-a3dc-6ca169c7e5d5 2021-11-11T20:57:17.801 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.802+0000 7fec80ff8380 20 RGWRados::pool_iterate: got zonegroup_info.83deb6b2-84e7-49ca-824d-ece614c4b869 2021-11-11T20:57:17.802 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.802+0000 7fec80ff8380 20 RGWRados::pool_iterate: got zone_names.default 2021-11-11T20:57:17.802 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.802+0000 7fec80ff8380 20 RGWRados::pool_iterate: got zonegroups_names.default 2021-11-11T20:57:17.802 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.802+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.803 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.802+0000 7fec80ff8380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:17.803 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.802+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.804 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.802+0000 7fec80ff8380 20 rados_obj.operate() r=0 bl.length=873 2021-11-11T20:57:17.805 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.802+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.805 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:17.805 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.806 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:17.806 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.806 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.806 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 20 zone default found 2021-11-11T20:57:17.806 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 4 Realm: () 2021-11-11T20:57:17.807 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 4 ZoneGroup: default (83deb6b2-84e7-49ca-824d-ece614c4b869) 2021-11-11T20:57:17.807 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 4 Zone: default (e42aa540-4a3f-4162-a3dc-6ca169c7e5d5) 2021-11-11T20:57:17.807 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 10 cannot find current period zonegroup using local zonegroup configuration 2021-11-11T20:57:17.807 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 20 zonegroup default 2021-11-11T20:57:17.808 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.803+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:17.808 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.804+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:17.808 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:17.804+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:19.762 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:19.763+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:19.763 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:19.763+0000 7fec80ff8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:19.763 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:19.764+0000 7fec80ff8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:19.764 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:19.764+0000 7fec80ff8380 20 started sync module instance, tier type = 2021-11-11T20:57:19.764 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:19.764+0000 7fec80ff8380 20 started zone id=e42aa540-4a3f-4162-a3dc-6ca169c7e5d5 (name=default) with tier type = 2021-11-11T20:57:21.808 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.808+0000 7fec80ff8380 20 add_watcher() i=0 2021-11-11T20:57:21.808 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.808+0000 7fec80ff8380 20 add_watcher() i=1 2021-11-11T20:57:21.808 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.808+0000 7fec80ff8380 20 add_watcher() i=2 2021-11-11T20:57:21.809 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.808+0000 7fec80ff8380 20 add_watcher() i=3 2021-11-11T20:57:21.809 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.808+0000 7fec80ff8380 20 add_watcher() i=4 2021-11-11T20:57:21.809 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.808+0000 7fec80ff8380 20 add_watcher() i=5 2021-11-11T20:57:21.809 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.808+0000 7fec80ff8380 20 add_watcher() i=6 2021-11-11T20:57:21.811 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.809+0000 7fec80ff8380 20 add_watcher() i=7 2021-11-11T20:57:21.812 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.809+0000 7fec80ff8380 2 all 8 watchers are set, enabling cache 2021-11-11T20:57:21.812 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.809+0000 7fec80ff8380 5 tl::expected, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):410: oid=data_loggenerations_metadata not found 2021-11-11T20:57:21.812 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.809+0000 7fec80ff8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.0 2021-11-11T20:57:21.812 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.809+0000 7fec80ff8380 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2021-11-11T20:57:21.813 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.810+0000 7fec80ff8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.0 does not exist 2021-11-11T20:57:21.813 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.810+0000 7fec80ff8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.1 2021-11-11T20:57:21.813 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.810+0000 7fec80ff8380 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2021-11-11T20:57:21.813 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.810+0000 7fec80ff8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.1 does not exist 2021-11-11T20:57:21.814 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.810+0000 7fec80ff8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.2 2021-11-11T20:57:21.814 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.810+0000 7fec80ff8380 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2021-11-11T20:57:21.814 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.810+0000 7fec80ff8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.2 does not exist 2021-11-11T20:57:21.814 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.810+0000 7fec80ff8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.3 2021-11-11T20:57:21.815 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.810+0000 7fec80ff8380 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2021-11-11T20:57:21.815 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.811+0000 7fec80ff8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.3 does not exist 2021-11-11T20:57:21.815 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.811+0000 7fec80ff8380 20 static int rgw::cls::fifo::FIFO::create(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, std::optional >, bool, uint64_t, uint64_t):1203 entering 2021-11-11T20:57:21.815 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.812+0000 7fec80ff8380 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2021-11-11T20:57:21.816 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.815+0000 7fec45fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2021-11-11T20:57:21.816 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.815+0000 7fec80ff8380 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2021-11-11T20:57:21.817 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:21.815+0000 7fec80ff8380 20 check_secure_mon_conn(): mode 1 is insecure 2021-11-11T20:57:25.790 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.790+0000 7fec80ff8380 10 rgw_init_ioctx warning: failed to set recovery_priority on default.rgw.meta 2021-11-11T20:57:25.790 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.790+0000 7fec80ff8380 5 note: GC not initialized 2021-11-11T20:57:25.791 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.792+0000 7febb4ff9700 20 reqs_thread_entry: start 2021-11-11T20:57:25.792 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.792+0000 7feba67fc700 20 reqs_thread_entry: start 2021-11-11T20:57:25.792 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.792+0000 7fec80ff8380 20 init_complete bucket index max shards: 11 2021-11-11T20:57:25.793 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.793+0000 7fec80ff8380 10 rgw notify: Started notification manager with: 1 workers 2021-11-11T20:57:25.793 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.793+0000 7feba4ff9700 20 rgw notify: INFO: next queues processing will happen at: Thu Nov 11 20:57:56 2021 2021-11-11T20:57:25.793 INFO:teuthology.orchestra.run.smithi096.stderr: 2021-11-11T20:57:25.794 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.793+0000 7feb8ffff700 20 reqs_thread_entry: start 2021-11-11T20:57:25.820 INFO:teuthology.orchestra.run.smithi096.stdout:[] 2021-11-11T20:57:25.821 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.821+0000 7fec80ff8380 20 remove_watcher() i=0 2021-11-11T20:57:25.821 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.821+0000 7fec80ff8380 2 removed watcher, disabling cache 2021-11-11T20:57:25.822 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.823+0000 7fec80ff8380 20 remove_watcher() i=1 2021-11-11T20:57:25.823 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.824+0000 7fec80ff8380 20 remove_watcher() i=2 2021-11-11T20:57:25.825 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.825+0000 7fec80ff8380 20 remove_watcher() i=3 2021-11-11T20:57:25.826 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.827+0000 7fec80ff8380 20 remove_watcher() i=4 2021-11-11T20:57:25.827 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.828+0000 7fec80ff8380 20 remove_watcher() i=5 2021-11-11T20:57:25.830 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.830+0000 7fec80ff8380 20 remove_watcher() i=6 2021-11-11T20:57:25.831 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.832+0000 7fec80ff8380 20 remove_watcher() i=7 2021-11-11T20:57:25.832 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.832+0000 7fec45fe3700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2021-11-11T20:57:25.837 DEBUG:tasks.util.rgw: json result: [] 2021-11-11T20:57:25.837 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2021-11-11T20:57:25.838 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2021-11-11T20:57:25.838 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2021-11-11T20:57:25.883 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:57:25.883 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:57:25.888 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.889+0000 7f4ae965a380 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2021-11-11T20:57:25.889 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.889+0000 7f4ae965a380 20 check_secure_mon_conn(): mode 1 is insecure 2021-11-11T20:57:25.892 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.893+0000 7f4a25ffb700 20 reqs_thread_entry: start 2021-11-11T20:57:25.892 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.893+0000 7f4ae965a380 20 try_to_resolve_local_entities(): before: realm_id= realm_name= zonegroup_id= zonegroup_name= zone_id= zone_name=default 2021-11-11T20:57:25.893 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.893+0000 7f4ae965a380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.893 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.894+0000 7f4ae965a380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:25.893 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.894+0000 7f4ae965a380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.894 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.895+0000 7f4ae965a380 20 rados_obj.operate() r=0 bl.length=873 2021-11-11T20:57:25.905 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.905+0000 7f4ae965a380 20 RGWRados::pool_iterate: got zone_info.e42aa540-4a3f-4162-a3dc-6ca169c7e5d5 2021-11-11T20:57:25.905 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.905+0000 7f4ae965a380 20 RGWRados::pool_iterate: got zonegroup_info.83deb6b2-84e7-49ca-824d-ece614c4b869 2021-11-11T20:57:25.905 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.905+0000 7f4ae965a380 20 RGWRados::pool_iterate: got zone_names.default 2021-11-11T20:57:25.906 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.905+0000 7f4ae965a380 20 RGWRados::pool_iterate: got zonegroups_names.default 2021-11-11T20:57:25.906 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.905+0000 7f4ae965a380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.906 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.905+0000 7f4ae965a380 20 rados_obj.operate() r=0 bl.length=873 2021-11-11T20:57:25.906 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.905+0000 7f4ae965a380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.907 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.906+0000 7f4ae965a380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:25.907 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.906+0000 7f4ae965a380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.908 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.906+0000 7f4ae965a380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:25.908 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.906+0000 7f4ae965a380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.908 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.906+0000 7f4ae965a380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:25.909 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T20:57:25.910 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "e42aa540-4a3f-4162-a3dc-6ca169c7e5d5", 2021-11-11T20:57:25.910 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "default", 2021-11-11T20:57:25.910 INFO:teuthology.orchestra.run.smithi096.stdout: "domain_root": "default.rgw.meta:root", 2021-11-11T20:57:25.910 INFO:teuthology.orchestra.run.smithi096.stdout: "control_pool": "default.rgw.control", 2021-11-11T20:57:25.911 INFO:teuthology.orchestra.run.smithi096.stdout: "gc_pool": "default.rgw.log:gc", 2021-11-11T20:57:25.911 INFO:teuthology.orchestra.run.smithi096.stdout: "lc_pool": "default.rgw.log:lc", 2021-11-11T20:57:25.911 INFO:teuthology.orchestra.run.smithi096.stdout: "log_pool": "default.rgw.log", 2021-11-11T20:57:25.911 INFO:teuthology.orchestra.run.smithi096.stdout: "intent_log_pool": "default.rgw.log:intent", 2021-11-11T20:57:25.912 INFO:teuthology.orchestra.run.smithi096.stdout: "usage_log_pool": "default.rgw.log:usage", 2021-11-11T20:57:25.912 INFO:teuthology.orchestra.run.smithi096.stdout: "roles_pool": "default.rgw.meta:roles", 2021-11-11T20:57:25.912 INFO:teuthology.orchestra.run.smithi096.stdout: "reshard_pool": "default.rgw.log:reshard", 2021-11-11T20:57:25.912 INFO:teuthology.orchestra.run.smithi096.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2021-11-11T20:57:25.913 INFO:teuthology.orchestra.run.smithi096.stdout: "user_email_pool": "default.rgw.meta:users.email", 2021-11-11T20:57:25.913 INFO:teuthology.orchestra.run.smithi096.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2021-11-11T20:57:25.913 INFO:teuthology.orchestra.run.smithi096.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2021-11-11T20:57:25.913 INFO:teuthology.orchestra.run.smithi096.stdout: "otp_pool": "default.rgw.otp", 2021-11-11T20:57:25.913 INFO:teuthology.orchestra.run.smithi096.stdout: "system_key": { 2021-11-11T20:57:25.914 INFO:teuthology.orchestra.run.smithi096.stdout: "access_key": "", 2021-11-11T20:57:25.914 INFO:teuthology.orchestra.run.smithi096.stdout: "secret_key": "" 2021-11-11T20:57:25.914 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T20:57:25.914 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_pools": [ 2021-11-11T20:57:25.915 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T20:57:25.915 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "default-placement", 2021-11-11T20:57:25.915 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T20:57:25.915 INFO:teuthology.orchestra.run.smithi096.stdout: "index_pool": "default.rgw.buckets.index", 2021-11-11T20:57:25.916 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": { 2021-11-11T20:57:25.916 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD": { 2021-11-11T20:57:25.916 INFO:teuthology.orchestra.run.smithi096.stdout: "data_pool": "default.rgw.buckets.data", 2021-11-11T20:57:25.916 INFO:teuthology.orchestra.run.smithi096.stdout: "compression_type": "random" 2021-11-11T20:57:25.917 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:57:25.917 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T20:57:25.917 INFO:teuthology.orchestra.run.smithi096.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2021-11-11T20:57:25.917 INFO:teuthology.orchestra.run.smithi096.stdout: "index_type": 0 2021-11-11T20:57:25.918 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:57:25.918 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:57:25.918 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T20:57:25.918 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "", 2021-11-11T20:57:25.919 INFO:teuthology.orchestra.run.smithi096.stdout: "notif_pool": "default.rgw.log:notif" 2021-11-11T20:57:25.919 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T20:57:25.920 DEBUG:tasks.util.rgw: json result: {'id': 'e42aa540-4a3f-4162-a3dc-6ca169c7e5d5', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2021-11-11T20:57:25.920 INFO:tasks.util.rgw:rgwadmin: c1.client.1 : ['user', 'list'] 2021-11-11T20:57:25.921 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c1', 'user', 'list'] 2021-11-11T20:57:25.921 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c1 user list 2021-11-11T20:57:25.958 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:57:25.958 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:57:25.964 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.964+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.965 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.966+0000 7fa4359a0380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:25.966 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.966+0000 7fa4359a0380 20 realm 2021-11-11T20:57:25.966 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.966+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.966 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.966+0000 7fa4359a0380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:25.966 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.966+0000 7fa4359a0380 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2021-11-11T20:57:25.967 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.966+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.967 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.966+0000 7fa4359a0380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:25.967 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.966+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.968 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.967+0000 7fa4359a0380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:25.968 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.967+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.970 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.970+0000 7fa4359a0380 20 rados_obj.operate() r=0 bl.length=889 2021-11-11T20:57:25.970 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.971+0000 7fa4359a0380 20 searching for the correct realm 2021-11-11T20:57:25.981 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.981+0000 7fa4359a0380 20 RGWRados::pool_iterate: got zone_info.e42aa540-4a3f-4162-a3dc-6ca169c7e5d5 2021-11-11T20:57:25.981 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.981+0000 7fa4359a0380 20 RGWRados::pool_iterate: got zonegroup_info.83deb6b2-84e7-49ca-824d-ece614c4b869 2021-11-11T20:57:25.981 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.981+0000 7fa4359a0380 20 RGWRados::pool_iterate: got zone_names.default 2021-11-11T20:57:25.982 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.981+0000 7fa4359a0380 20 RGWRados::pool_iterate: got zonegroups_names.default 2021-11-11T20:57:25.982 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.981+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.982 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.981+0000 7fa4359a0380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:25.982 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.981+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.983 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.982+0000 7fa4359a0380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:25.983 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.982+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.984 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.982+0000 7fa4359a0380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:25.984 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.982+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.984 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.982+0000 7fa4359a0380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:25.985 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.982+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.985 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.983+0000 7fa4359a0380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:25.985 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.983+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.985 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.983+0000 7fa4359a0380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:25.992 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.992+0000 7fa4359a0380 20 RGWRados::pool_iterate: got zone_info.e42aa540-4a3f-4162-a3dc-6ca169c7e5d5 2021-11-11T20:57:25.992 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.992+0000 7fa4359a0380 20 RGWRados::pool_iterate: got zonegroup_info.83deb6b2-84e7-49ca-824d-ece614c4b869 2021-11-11T20:57:25.992 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.992+0000 7fa4359a0380 20 RGWRados::pool_iterate: got zone_names.default 2021-11-11T20:57:25.992 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.992+0000 7fa4359a0380 20 RGWRados::pool_iterate: got zonegroups_names.default 2021-11-11T20:57:25.993 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.992+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.993 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.993+0000 7fa4359a0380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:25.993 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.993+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.995 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.993+0000 7fa4359a0380 20 rados_obj.operate() r=0 bl.length=889 2021-11-11T20:57:25.995 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.993+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.995 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.993+0000 7fa4359a0380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:25.996 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.993+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.996 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:25.996 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.996 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:25.997 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 20 zone default found 2021-11-11T20:57:25.997 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 4 Realm: () 2021-11-11T20:57:25.997 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 4 ZoneGroup: default (83deb6b2-84e7-49ca-824d-ece614c4b869) 2021-11-11T20:57:25.997 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 4 Zone: default (e42aa540-4a3f-4162-a3dc-6ca169c7e5d5) 2021-11-11T20:57:25.998 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 10 cannot find current period zonegroup using local zonegroup configuration 2021-11-11T20:57:25.998 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 20 zonegroup default 2021-11-11T20:57:25.998 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.998 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:25.999 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.994+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.999 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.995+0000 7fa4359a0380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:25.999 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.995+0000 7fa4359a0380 20 rados->read ofs=0 len=0 2021-11-11T20:57:25.999 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.995+0000 7fa4359a0380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:25.999 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.995+0000 7fa4359a0380 20 started sync module instance, tier type = 2021-11-11T20:57:26.000 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:25.995+0000 7fa4359a0380 20 started zone id=e42aa540-4a3f-4162-a3dc-6ca169c7e5d5 (name=default) with tier type = 2021-11-11T20:57:26.010 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.011+0000 7fa4359a0380 20 add_watcher() i=0 2021-11-11T20:57:26.011 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.011+0000 7fa4359a0380 20 add_watcher() i=1 2021-11-11T20:57:26.011 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.011+0000 7fa4359a0380 20 add_watcher() i=2 2021-11-11T20:57:26.011 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.011+0000 7fa4359a0380 20 add_watcher() i=3 2021-11-11T20:57:26.011 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.011+0000 7fa4359a0380 20 add_watcher() i=4 2021-11-11T20:57:26.012 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.011+0000 7fa4359a0380 20 add_watcher() i=5 2021-11-11T20:57:26.012 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.011+0000 7fa4359a0380 20 add_watcher() i=6 2021-11-11T20:57:26.012 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.012+0000 7fa4359a0380 20 add_watcher() i=7 2021-11-11T20:57:26.013 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.012+0000 7fa4359a0380 2 all 8 watchers are set, enabling cache 2021-11-11T20:57:26.013 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.014+0000 7fa3fa7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2021-11-11T20:57:26.014 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.014+0000 7fa4359a0380 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2021-11-11T20:57:26.014 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.014+0000 7fa4359a0380 20 check_secure_mon_conn(): mode 1 is insecure 2021-11-11T20:57:26.017 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.018+0000 7fa4359a0380 5 note: GC not initialized 2021-11-11T20:57:26.018 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.019+0000 7fa3697fa700 20 reqs_thread_entry: start 2021-11-11T20:57:26.019 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.019+0000 7fa362ffd700 20 reqs_thread_entry: start 2021-11-11T20:57:26.019 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.019+0000 7fa4359a0380 20 init_complete bucket index max shards: 11 2021-11-11T20:57:26.019 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.019+0000 7fa4359a0380 10 rgw notify: Started notification manager with: 1 workers 2021-11-11T20:57:26.019 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.019+0000 7fa3617fa700 20 rgw notify: INFO: next queues processing will happen at: Thu Nov 11 20:57:56 2021 2021-11-11T20:57:26.020 INFO:teuthology.orchestra.run.smithi096.stderr: 2021-11-11T20:57:26.020 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.019+0000 7fa360ff9700 20 reqs_thread_entry: start 2021-11-11T20:57:26.030 INFO:teuthology.orchestra.run.smithi096.stdout:[] 2021-11-11T20:57:26.031 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.031+0000 7fa4359a0380 20 remove_watcher() i=0 2021-11-11T20:57:26.031 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.031+0000 7fa4359a0380 2 removed watcher, disabling cache 2021-11-11T20:57:26.032 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.032+0000 7fa4359a0380 20 remove_watcher() i=1 2021-11-11T20:57:26.033 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.033+0000 7fa4359a0380 20 remove_watcher() i=2 2021-11-11T20:57:26.034 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.035+0000 7fa4359a0380 20 remove_watcher() i=3 2021-11-11T20:57:26.035 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.036+0000 7fa4359a0380 20 remove_watcher() i=4 2021-11-11T20:57:26.036 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.037+0000 7fa4359a0380 20 remove_watcher() i=5 2021-11-11T20:57:26.037 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.038+0000 7fa4359a0380 20 remove_watcher() i=6 2021-11-11T20:57:26.038 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.039+0000 7fa4359a0380 20 remove_watcher() i=7 2021-11-11T20:57:26.039 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.040+0000 7fa3fa7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2021-11-11T20:57:26.044 DEBUG:tasks.util.rgw: json result: [] 2021-11-11T20:57:26.044 INFO:tasks.util.rgw:rgwadmin: c1.client.1 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2021-11-11T20:57:26.044 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c1', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2021-11-11T20:57:26.045 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c1 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2021-11-11T20:57:26.126 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:57:26.126 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:57:26.131 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.132+0000 7f2a4b594380 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2021-11-11T20:57:26.131 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.132+0000 7f2a4b594380 20 check_secure_mon_conn(): mode 1 is insecure 2021-11-11T20:57:26.135 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.135+0000 7f2987fff700 20 reqs_thread_entry: start 2021-11-11T20:57:26.135 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.135+0000 7f2a4b594380 20 try_to_resolve_local_entities(): before: realm_id= realm_name= zonegroup_id= zonegroup_name= zone_id= zone_name=default 2021-11-11T20:57:26.135 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.135+0000 7f2a4b594380 20 rados->read ofs=0 len=0 2021-11-11T20:57:26.136 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.136+0000 7f2a4b594380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:26.136 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.136+0000 7f2a4b594380 20 rados->read ofs=0 len=0 2021-11-11T20:57:26.137 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.137+0000 7f2a4b594380 20 rados_obj.operate() r=0 bl.length=889 2021-11-11T20:57:26.147 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.147+0000 7f2a4b594380 20 RGWRados::pool_iterate: got zone_info.e42aa540-4a3f-4162-a3dc-6ca169c7e5d5 2021-11-11T20:57:26.147 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.147+0000 7f2a4b594380 20 RGWRados::pool_iterate: got zonegroup_info.83deb6b2-84e7-49ca-824d-ece614c4b869 2021-11-11T20:57:26.147 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.147+0000 7f2a4b594380 20 RGWRados::pool_iterate: got zone_names.default 2021-11-11T20:57:26.148 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.147+0000 7f2a4b594380 20 RGWRados::pool_iterate: got zonegroups_names.default 2021-11-11T20:57:26.148 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.147+0000 7f2a4b594380 20 rados->read ofs=0 len=0 2021-11-11T20:57:26.148 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.147+0000 7f2a4b594380 20 rados_obj.operate() r=0 bl.length=889 2021-11-11T20:57:26.148 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.147+0000 7f2a4b594380 20 rados->read ofs=0 len=0 2021-11-11T20:57:26.149 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.147+0000 7f2a4b594380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:26.149 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.148+0000 7f2a4b594380 20 rados->read ofs=0 len=0 2021-11-11T20:57:26.150 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.148+0000 7f2a4b594380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:26.150 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.148+0000 7f2a4b594380 20 rados->read ofs=0 len=0 2021-11-11T20:57:26.150 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:57:26.148+0000 7f2a4b594380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:26.151 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T20:57:26.152 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "e42aa540-4a3f-4162-a3dc-6ca169c7e5d5", 2021-11-11T20:57:26.152 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "default", 2021-11-11T20:57:26.152 INFO:teuthology.orchestra.run.smithi096.stdout: "domain_root": "default.rgw.meta:root", 2021-11-11T20:57:26.153 INFO:teuthology.orchestra.run.smithi096.stdout: "control_pool": "default.rgw.control", 2021-11-11T20:57:26.153 INFO:teuthology.orchestra.run.smithi096.stdout: "gc_pool": "default.rgw.log:gc", 2021-11-11T20:57:26.153 INFO:teuthology.orchestra.run.smithi096.stdout: "lc_pool": "default.rgw.log:lc", 2021-11-11T20:57:26.153 INFO:teuthology.orchestra.run.smithi096.stdout: "log_pool": "default.rgw.log", 2021-11-11T20:57:26.154 INFO:teuthology.orchestra.run.smithi096.stdout: "intent_log_pool": "default.rgw.log:intent", 2021-11-11T20:57:26.154 INFO:teuthology.orchestra.run.smithi096.stdout: "usage_log_pool": "default.rgw.log:usage", 2021-11-11T20:57:26.154 INFO:teuthology.orchestra.run.smithi096.stdout: "roles_pool": "default.rgw.meta:roles", 2021-11-11T20:57:26.154 INFO:teuthology.orchestra.run.smithi096.stdout: "reshard_pool": "default.rgw.log:reshard", 2021-11-11T20:57:26.155 INFO:teuthology.orchestra.run.smithi096.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2021-11-11T20:57:26.155 INFO:teuthology.orchestra.run.smithi096.stdout: "user_email_pool": "default.rgw.meta:users.email", 2021-11-11T20:57:26.155 INFO:teuthology.orchestra.run.smithi096.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2021-11-11T20:57:26.155 INFO:teuthology.orchestra.run.smithi096.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2021-11-11T20:57:26.156 INFO:teuthology.orchestra.run.smithi096.stdout: "otp_pool": "default.rgw.otp", 2021-11-11T20:57:26.156 INFO:teuthology.orchestra.run.smithi096.stdout: "system_key": { 2021-11-11T20:57:26.156 INFO:teuthology.orchestra.run.smithi096.stdout: "access_key": "", 2021-11-11T20:57:26.156 INFO:teuthology.orchestra.run.smithi096.stdout: "secret_key": "" 2021-11-11T20:57:26.157 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T20:57:26.157 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_pools": [ 2021-11-11T20:57:26.157 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T20:57:26.157 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "default-placement", 2021-11-11T20:57:26.158 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T20:57:26.158 INFO:teuthology.orchestra.run.smithi096.stdout: "index_pool": "default.rgw.buckets.index", 2021-11-11T20:57:26.158 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": { 2021-11-11T20:57:26.158 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD": { 2021-11-11T20:57:26.159 INFO:teuthology.orchestra.run.smithi096.stdout: "data_pool": "default.rgw.buckets.data", 2021-11-11T20:57:26.159 INFO:teuthology.orchestra.run.smithi096.stdout: "compression_type": "random" 2021-11-11T20:57:26.159 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:57:26.159 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T20:57:26.160 INFO:teuthology.orchestra.run.smithi096.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2021-11-11T20:57:26.160 INFO:teuthology.orchestra.run.smithi096.stdout: "index_type": 0 2021-11-11T20:57:26.160 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:57:26.160 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:57:26.160 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T20:57:26.161 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "", 2021-11-11T20:57:26.161 INFO:teuthology.orchestra.run.smithi096.stdout: "notif_pool": "default.rgw.log:notif" 2021-11-11T20:57:26.161 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T20:57:26.162 DEBUG:tasks.util.rgw: json result: {'id': 'e42aa540-4a3f-4162-a3dc-6ca169c7e5d5', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2021-11-11T20:57:26.163 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['user', 'list'] 2021-11-11T20:57:26.163 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'user', 'list'] 2021-11-11T20:57:26.164 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 user list 2021-11-11T20:57:26.235 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:57:26.236 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:57:26.242 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:26.243+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.405 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.405+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.406 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.405+0000 7fe6065b8380 20 realm 2021-11-11T20:57:27.406 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.405+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.407 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.407+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.407 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.407+0000 7fe6065b8380 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2021-11-11T20:57:27.408 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.407+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.408 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.408+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.409 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.408+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.409 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.409+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.409 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.409+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.410 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.410+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.410 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.410+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.411 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.410+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.411 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.410+0000 7fe6065b8380 10 Creating default zonegroup 2021-11-11T20:57:27.411 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.410+0000 7fe6065b8380 20 get_system_obj_state: rctx=0x7ffffcf453b0 obj=:.pools.avail state=0x55fb27f06eb0 s->prefetch_data=0 2021-11-11T20:57:27.412 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.411+0000 7fe6065b8380 10 couldn't find old data placement pools config, setting up new ones for the zone 2021-11-11T20:57:27.428 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.428+0000 7fe6065b8380 10 WARNING: store->list_zones() returned r=-2 2021-11-11T20:57:27.428 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.428+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.429 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.429+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.432 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.432+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.432 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.432+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.432 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.432+0000 7fe6065b8380 10 could not read realm id: (2) No such file or directory 2021-11-11T20:57:27.432 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.432+0000 7fe6065b8380 10 WARNING: failed to set zone as default, r=-22 2021-11-11T20:57:27.433 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.432+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.433 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.432+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.435 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.435+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.435 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.436+0000 7fe6065b8380 20 rados_obj.operate() r=0 bl.length=873 2021-11-11T20:57:27.436 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.436+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.437 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.436+0000 7fe6065b8380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:27.437 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.436+0000 7fe6065b8380 10 Using default name default 2021-11-11T20:57:27.438 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.436+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.438 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.437+0000 7fe6065b8380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:27.438 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.437+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.438 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.437+0000 7fe6065b8380 20 rados_obj.operate() r=0 bl.length=873 2021-11-11T20:57:27.439 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.437+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.439 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.438+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.439 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.438+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.440 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.438+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.440 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.438+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.440 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.438+0000 7fe6065b8380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:27.449 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.449+0000 7fe6065b8380 20 RGWRados::pool_iterate: got zone_info.fda9d225-837d-46c3-b3f5-759647d0b924 2021-11-11T20:57:27.449 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.449+0000 7fe6065b8380 20 RGWRados::pool_iterate: got zonegroup_info.fd16e0e2-3d65-4610-ba66-92e4fc5ea00a 2021-11-11T20:57:27.450 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.449+0000 7fe6065b8380 20 RGWRados::pool_iterate: got zone_names.default 2021-11-11T20:57:27.450 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.449+0000 7fe6065b8380 20 RGWRados::pool_iterate: got zonegroups_names.default 2021-11-11T20:57:27.450 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.449+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.450 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.450+0000 7fe6065b8380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:27.451 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.450+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.452 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.450+0000 7fe6065b8380 20 rados_obj.operate() r=0 bl.length=873 2021-11-11T20:57:27.453 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.450+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.453 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.450+0000 7fe6065b8380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:27.453 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.450+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.454 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:27.454 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.454 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.454 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 20 zone default found 2021-11-11T20:57:27.455 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 4 Realm: () 2021-11-11T20:57:27.455 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 4 ZoneGroup: default (fd16e0e2-3d65-4610-ba66-92e4fc5ea00a) 2021-11-11T20:57:27.455 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 4 Zone: default (fda9d225-837d-46c3-b3f5-759647d0b924) 2021-11-11T20:57:27.455 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 10 cannot find current period zonegroup using local zonegroup configuration 2021-11-11T20:57:27.456 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 20 zonegroup default 2021-11-11T20:57:27.456 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:27.456 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:27.456 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:27.451+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:29.417 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:29.416+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:29.417 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:29.416+0000 7fe6065b8380 20 rados->read ofs=0 len=0 2021-11-11T20:57:29.417 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:29.417+0000 7fe6065b8380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:29.418 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:29.417+0000 7fe6065b8380 20 started sync module instance, tier type = 2021-11-11T20:57:29.418 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:29.417+0000 7fe6065b8380 20 started zone id=fda9d225-837d-46c3-b3f5-759647d0b924 (name=default) with tier type = 2021-11-11T20:57:31.474 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.474+0000 7fe6065b8380 20 add_watcher() i=0 2021-11-11T20:57:31.475 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.474+0000 7fe6065b8380 20 add_watcher() i=1 2021-11-11T20:57:31.475 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.474+0000 7fe6065b8380 20 add_watcher() i=2 2021-11-11T20:57:31.476 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.474+0000 7fe6065b8380 20 add_watcher() i=3 2021-11-11T20:57:31.476 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.474+0000 7fe6065b8380 20 add_watcher() i=4 2021-11-11T20:57:31.477 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.474+0000 7fe6065b8380 20 add_watcher() i=5 2021-11-11T20:57:31.477 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.474+0000 7fe6065b8380 20 add_watcher() i=6 2021-11-11T20:57:31.479 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.475+0000 7fe6065b8380 20 add_watcher() i=7 2021-11-11T20:57:31.479 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.475+0000 7fe6065b8380 2 all 8 watchers are set, enabling cache 2021-11-11T20:57:31.480 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.476+0000 7fe6065b8380 5 tl::expected, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):410: oid=data_loggenerations_metadata not found 2021-11-11T20:57:31.480 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.476+0000 7fe6065b8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.0 2021-11-11T20:57:31.480 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.476+0000 7fe6065b8380 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2021-11-11T20:57:31.480 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.476+0000 7fe6065b8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.0 does not exist 2021-11-11T20:57:31.481 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.476+0000 7fe6065b8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.1 2021-11-11T20:57:31.481 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.476+0000 7fe6065b8380 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2021-11-11T20:57:31.481 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.476+0000 7fe6065b8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.1 does not exist 2021-11-11T20:57:31.481 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.476+0000 7fe6065b8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.2 2021-11-11T20:57:31.482 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.476+0000 7fe6065b8380 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2021-11-11T20:57:31.482 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.477+0000 7fe6065b8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.2 does not exist 2021-11-11T20:57:31.482 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.477+0000 7fe6065b8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):38 probing oid=data_log.3 2021-11-11T20:57:31.482 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.477+0000 7fe6065b8380 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2021-11-11T20:57:31.483 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.477+0000 7fe6065b8380 20 shard_check {anonymous}::probe_shard(const DoutPrefixProvider*, librados::v14_2_0::IoCtx&, const string&, bool&, optional_yield):60: oid=data_log.3 does not exist 2021-11-11T20:57:31.483 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.477+0000 7fe6065b8380 20 static int rgw::cls::fifo::FIFO::create(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, std::optional >, bool, uint64_t, uint64_t):1203 entering 2021-11-11T20:57:31.483 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.479+0000 7fe6065b8380 20 static int rgw::cls::fifo::FIFO::open(const DoutPrefixProvider*, librados::v14_2_0::IoCtx, std::__cxx11::string, std::unique_ptr*, optional_yield, std::optional, bool):1161 entering 2021-11-11T20:57:31.484 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.484+0000 7fe5ca7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2021-11-11T20:57:31.484 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.484+0000 7fe6065b8380 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2021-11-11T20:57:31.484 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:31.484+0000 7fe6065b8380 20 check_secure_mon_conn(): mode 1 is insecure 2021-11-11T20:57:35.442 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.442+0000 7fe6065b8380 10 rgw_init_ioctx warning: failed to set recovery_priority on default.rgw.meta 2021-11-11T20:57:35.442 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.442+0000 7fe6065b8380 5 note: GC not initialized 2021-11-11T20:57:35.444 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.444+0000 7fe5397fa700 20 reqs_thread_entry: start 2021-11-11T20:57:35.444 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.444+0000 7fe6065b8380 20 init_complete bucket index max shards: 11 2021-11-11T20:57:35.445 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.445+0000 7fe6065b8380 10 rgw notify: Started notification manager with: 1 workers 2021-11-11T20:57:35.445 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.445+0000 7fe526ffd700 20 reqs_thread_entry: start 2021-11-11T20:57:35.445 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.445+0000 7fe524ff9700 20 reqs_thread_entry: start 2021-11-11T20:57:35.446 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.445+0000 7fe5257fa700 20 rgw notify: INFO: next queues processing will happen at: Thu Nov 11 20:58:05 2021 2021-11-11T20:57:35.446 INFO:teuthology.orchestra.run.smithi198.stderr: 2021-11-11T20:57:35.474 INFO:teuthology.orchestra.run.smithi198.stdout:[] 2021-11-11T20:57:35.475 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.474+0000 7fe6065b8380 20 remove_watcher() i=0 2021-11-11T20:57:35.475 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.474+0000 7fe6065b8380 2 removed watcher, disabling cache 2021-11-11T20:57:35.475 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.476+0000 7fe6065b8380 20 remove_watcher() i=1 2021-11-11T20:57:35.477 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.477+0000 7fe6065b8380 20 remove_watcher() i=2 2021-11-11T20:57:35.478 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.478+0000 7fe6065b8380 20 remove_watcher() i=3 2021-11-11T20:57:35.479 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.480+0000 7fe6065b8380 20 remove_watcher() i=4 2021-11-11T20:57:35.480 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.481+0000 7fe6065b8380 20 remove_watcher() i=5 2021-11-11T20:57:35.482 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.482+0000 7fe6065b8380 20 remove_watcher() i=6 2021-11-11T20:57:35.483 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.484+0000 7fe6065b8380 20 remove_watcher() i=7 2021-11-11T20:57:35.484 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.484+0000 7fe5ca7e4700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2021-11-11T20:57:35.490 DEBUG:tasks.util.rgw: json result: [] 2021-11-11T20:57:35.490 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2021-11-11T20:57:35.491 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2021-11-11T20:57:35.491 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2021-11-11T20:57:35.536 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:57:35.536 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:57:35.542 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.542+0000 7fc3a591f380 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2021-11-11T20:57:35.542 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.542+0000 7fc3a591f380 20 check_secure_mon_conn(): mode 1 is insecure 2021-11-11T20:57:35.545 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.545+0000 7fc2e27fc700 20 reqs_thread_entry: start 2021-11-11T20:57:35.545 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.545+0000 7fc3a591f380 20 try_to_resolve_local_entities(): before: realm_id= realm_name= zonegroup_id= zonegroup_name= zone_id= zone_name=default 2021-11-11T20:57:35.545 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.545+0000 7fc3a591f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.546 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.546+0000 7fc3a591f380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:35.546 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.546+0000 7fc3a591f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.547 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.547+0000 7fc3a591f380 20 rados_obj.operate() r=0 bl.length=873 2021-11-11T20:57:35.558 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.557+0000 7fc3a591f380 20 RGWRados::pool_iterate: got zone_info.fda9d225-837d-46c3-b3f5-759647d0b924 2021-11-11T20:57:35.558 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.557+0000 7fc3a591f380 20 RGWRados::pool_iterate: got zonegroup_info.fd16e0e2-3d65-4610-ba66-92e4fc5ea00a 2021-11-11T20:57:35.559 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.557+0000 7fc3a591f380 20 RGWRados::pool_iterate: got zone_names.default 2021-11-11T20:57:35.559 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.557+0000 7fc3a591f380 20 RGWRados::pool_iterate: got zonegroups_names.default 2021-11-11T20:57:35.559 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.557+0000 7fc3a591f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.559 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.558+0000 7fc3a591f380 20 rados_obj.operate() r=0 bl.length=873 2021-11-11T20:57:35.559 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.558+0000 7fc3a591f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.560 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.558+0000 7fc3a591f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.560 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.558+0000 7fc3a591f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.560 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.558+0000 7fc3a591f380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:35.560 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.558+0000 7fc3a591f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.561 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.559+0000 7fc3a591f380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:35.562 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T20:57:35.563 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "fda9d225-837d-46c3-b3f5-759647d0b924", 2021-11-11T20:57:35.563 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "default", 2021-11-11T20:57:35.563 INFO:teuthology.orchestra.run.smithi198.stdout: "domain_root": "default.rgw.meta:root", 2021-11-11T20:57:35.563 INFO:teuthology.orchestra.run.smithi198.stdout: "control_pool": "default.rgw.control", 2021-11-11T20:57:35.564 INFO:teuthology.orchestra.run.smithi198.stdout: "gc_pool": "default.rgw.log:gc", 2021-11-11T20:57:35.564 INFO:teuthology.orchestra.run.smithi198.stdout: "lc_pool": "default.rgw.log:lc", 2021-11-11T20:57:35.564 INFO:teuthology.orchestra.run.smithi198.stdout: "log_pool": "default.rgw.log", 2021-11-11T20:57:35.564 INFO:teuthology.orchestra.run.smithi198.stdout: "intent_log_pool": "default.rgw.log:intent", 2021-11-11T20:57:35.564 INFO:teuthology.orchestra.run.smithi198.stdout: "usage_log_pool": "default.rgw.log:usage", 2021-11-11T20:57:35.565 INFO:teuthology.orchestra.run.smithi198.stdout: "roles_pool": "default.rgw.meta:roles", 2021-11-11T20:57:35.565 INFO:teuthology.orchestra.run.smithi198.stdout: "reshard_pool": "default.rgw.log:reshard", 2021-11-11T20:57:35.565 INFO:teuthology.orchestra.run.smithi198.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2021-11-11T20:57:35.565 INFO:teuthology.orchestra.run.smithi198.stdout: "user_email_pool": "default.rgw.meta:users.email", 2021-11-11T20:57:35.566 INFO:teuthology.orchestra.run.smithi198.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2021-11-11T20:57:35.566 INFO:teuthology.orchestra.run.smithi198.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2021-11-11T20:57:35.566 INFO:teuthology.orchestra.run.smithi198.stdout: "otp_pool": "default.rgw.otp", 2021-11-11T20:57:35.566 INFO:teuthology.orchestra.run.smithi198.stdout: "system_key": { 2021-11-11T20:57:35.566 INFO:teuthology.orchestra.run.smithi198.stdout: "access_key": "", 2021-11-11T20:57:35.567 INFO:teuthology.orchestra.run.smithi198.stdout: "secret_key": "" 2021-11-11T20:57:35.567 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T20:57:35.567 INFO:teuthology.orchestra.run.smithi198.stdout: "placement_pools": [ 2021-11-11T20:57:35.567 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T20:57:35.568 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "default-placement", 2021-11-11T20:57:35.568 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T20:57:35.568 INFO:teuthology.orchestra.run.smithi198.stdout: "index_pool": "default.rgw.buckets.index", 2021-11-11T20:57:35.568 INFO:teuthology.orchestra.run.smithi198.stdout: "storage_classes": { 2021-11-11T20:57:35.568 INFO:teuthology.orchestra.run.smithi198.stdout: "STANDARD": { 2021-11-11T20:57:35.569 INFO:teuthology.orchestra.run.smithi198.stdout: "data_pool": "default.rgw.buckets.data", 2021-11-11T20:57:35.569 INFO:teuthology.orchestra.run.smithi198.stdout: "compression_type": "random" 2021-11-11T20:57:35.569 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T20:57:35.569 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T20:57:35.570 INFO:teuthology.orchestra.run.smithi198.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2021-11-11T20:57:35.570 INFO:teuthology.orchestra.run.smithi198.stdout: "index_type": 0 2021-11-11T20:57:35.570 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T20:57:35.570 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T20:57:35.570 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T20:57:35.571 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "", 2021-11-11T20:57:35.571 INFO:teuthology.orchestra.run.smithi198.stdout: "notif_pool": "default.rgw.log:notif" 2021-11-11T20:57:35.571 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T20:57:35.572 DEBUG:tasks.util.rgw: json result: {'id': 'fda9d225-837d-46c3-b3f5-759647d0b924', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2021-11-11T20:57:35.573 INFO:tasks.util.rgw:rgwadmin: c2.client.1 : ['user', 'list'] 2021-11-11T20:57:35.573 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c2', 'user', 'list'] 2021-11-11T20:57:35.574 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c2 user list 2021-11-11T20:57:35.611 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:57:35.612 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:57:35.617 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.617+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.618 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.618+0000 7f616bc8f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.619 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.619+0000 7f616bc8f380 20 realm 2021-11-11T20:57:35.619 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.619+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.619 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.619+0000 7f616bc8f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.619 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.619+0000 7f616bc8f380 4 RGWPeriod::init failed to init realm id : (2) No such file or directory 2021-11-11T20:57:35.620 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.619+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.620 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.619+0000 7f616bc8f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.620 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.619+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.622 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.619+0000 7f616bc8f380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:35.622 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.619+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.622 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.620+0000 7f616bc8f380 20 rados_obj.operate() r=0 bl.length=889 2021-11-11T20:57:35.622 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.620+0000 7f616bc8f380 20 searching for the correct realm 2021-11-11T20:57:35.630 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.630+0000 7f616bc8f380 20 RGWRados::pool_iterate: got zone_info.fda9d225-837d-46c3-b3f5-759647d0b924 2021-11-11T20:57:35.630 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.630+0000 7f616bc8f380 20 RGWRados::pool_iterate: got zonegroup_info.fd16e0e2-3d65-4610-ba66-92e4fc5ea00a 2021-11-11T20:57:35.630 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.630+0000 7f616bc8f380 20 RGWRados::pool_iterate: got zone_names.default 2021-11-11T20:57:35.631 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.630+0000 7f616bc8f380 20 RGWRados::pool_iterate: got zonegroups_names.default 2021-11-11T20:57:35.631 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.630+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.631 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.630+0000 7f616bc8f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.631 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.630+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.632 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.630+0000 7f616bc8f380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:35.632 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.630+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.633 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.631+0000 7f616bc8f380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:35.633 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.631+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.634 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.631+0000 7f616bc8f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.634 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.631+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.634 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.631+0000 7f616bc8f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.634 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.631+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.635 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.632+0000 7f616bc8f380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:35.640 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.640+0000 7f616bc8f380 20 RGWRados::pool_iterate: got zone_info.fda9d225-837d-46c3-b3f5-759647d0b924 2021-11-11T20:57:35.641 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.640+0000 7f616bc8f380 20 RGWRados::pool_iterate: got zonegroup_info.fd16e0e2-3d65-4610-ba66-92e4fc5ea00a 2021-11-11T20:57:35.641 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.640+0000 7f616bc8f380 20 RGWRados::pool_iterate: got zone_names.default 2021-11-11T20:57:35.641 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.640+0000 7f616bc8f380 20 RGWRados::pool_iterate: got zonegroups_names.default 2021-11-11T20:57:35.641 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.640+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.642 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.640+0000 7f616bc8f380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:35.642 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.640+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.642 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.641+0000 7f616bc8f380 20 rados_obj.operate() r=0 bl.length=889 2021-11-11T20:57:35.642 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.641+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.642 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.641+0000 7f616bc8f380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:35.643 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.641+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.645 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.641+0000 7f616bc8f380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:35.645 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.641+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.645 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.645 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 20 zone default found 2021-11-11T20:57:35.646 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 4 Realm: () 2021-11-11T20:57:35.646 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 4 ZoneGroup: default (fd16e0e2-3d65-4610-ba66-92e4fc5ea00a) 2021-11-11T20:57:35.646 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 4 Zone: default (fda9d225-837d-46c3-b3f5-759647d0b924) 2021-11-11T20:57:35.646 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 10 cannot find current period zonegroup using local zonegroup configuration 2021-11-11T20:57:35.647 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 20 zonegroup default 2021-11-11T20:57:35.647 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.647 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.647 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.648 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.648 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.648 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.648 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 20 started sync module instance, tier type = 2021-11-11T20:57:35.648 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.642+0000 7f616bc8f380 20 started zone id=fda9d225-837d-46c3-b3f5-759647d0b924 (name=default) with tier type = 2021-11-11T20:57:35.652 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.653+0000 7f616bc8f380 20 add_watcher() i=0 2021-11-11T20:57:35.653 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.653+0000 7f616bc8f380 20 add_watcher() i=1 2021-11-11T20:57:35.653 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.653+0000 7f616bc8f380 20 add_watcher() i=2 2021-11-11T20:57:35.653 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.653+0000 7f616bc8f380 20 add_watcher() i=3 2021-11-11T20:57:35.653 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.653+0000 7f616bc8f380 20 add_watcher() i=4 2021-11-11T20:57:35.654 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.653+0000 7f616bc8f380 20 add_watcher() i=5 2021-11-11T20:57:35.654 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.653+0000 7f616bc8f380 20 add_watcher() i=6 2021-11-11T20:57:35.654 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.654+0000 7f616bc8f380 20 add_watcher() i=7 2021-11-11T20:57:35.655 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.654+0000 7f616bc8f380 2 all 8 watchers are set, enabling cache 2021-11-11T20:57:35.656 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.656+0000 7f612ffe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2021-11-11T20:57:35.656 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.656+0000 7f616bc8f380 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2021-11-11T20:57:35.656 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.656+0000 7f616bc8f380 20 check_secure_mon_conn(): mode 1 is insecure 2021-11-11T20:57:35.659 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.659+0000 7f616bc8f380 5 note: GC not initialized 2021-11-11T20:57:35.660 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.660+0000 7f609effd700 20 reqs_thread_entry: start 2021-11-11T20:57:35.660 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.660+0000 7f609cff9700 20 reqs_thread_entry: start 2021-11-11T20:57:35.661 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.660+0000 7f616bc8f380 20 init_complete bucket index max shards: 11 2021-11-11T20:57:35.661 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.661+0000 7f616bc8f380 10 rgw notify: Started notification manager with: 1 workers 2021-11-11T20:57:35.661 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.661+0000 7f608effd700 20 rgw notify: INFO: next queues processing will happen at: Thu Nov 11 20:58:06 2021 2021-11-11T20:57:35.661 INFO:teuthology.orchestra.run.smithi198.stderr: 2021-11-11T20:57:35.662 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.661+0000 7f608e7fc700 20 reqs_thread_entry: start 2021-11-11T20:57:35.672 INFO:teuthology.orchestra.run.smithi198.stdout:[] 2021-11-11T20:57:35.672 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.672+0000 7f616bc8f380 20 remove_watcher() i=0 2021-11-11T20:57:35.672 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.672+0000 7f616bc8f380 2 removed watcher, disabling cache 2021-11-11T20:57:35.673 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.674+0000 7f616bc8f380 20 remove_watcher() i=1 2021-11-11T20:57:35.674 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.675+0000 7f616bc8f380 20 remove_watcher() i=2 2021-11-11T20:57:35.677 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.677+0000 7f616bc8f380 20 remove_watcher() i=3 2021-11-11T20:57:35.679 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.680+0000 7f616bc8f380 20 remove_watcher() i=4 2021-11-11T20:57:35.680 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.681+0000 7f616bc8f380 20 remove_watcher() i=5 2021-11-11T20:57:35.682 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.682+0000 7f616bc8f380 20 remove_watcher() i=6 2021-11-11T20:57:35.684 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.685+0000 7f616bc8f380 20 remove_watcher() i=7 2021-11-11T20:57:35.685 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.686+0000 7f612ffe7700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2021-11-11T20:57:35.690 DEBUG:tasks.util.rgw: json result: [] 2021-11-11T20:57:35.690 INFO:tasks.util.rgw:rgwadmin: c2.client.1 : ['zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2021-11-11T20:57:35.691 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.1', '--cluster', 'c2', 'zone', 'placement', 'modify', '--rgw-zone', 'default', '--placement-id', 'default-placement', '--compression', 'random'] 2021-11-11T20:57:35.691 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.1 --cluster c2 zone placement modify --rgw-zone default --placement-id default-placement --compression random 2021-11-11T20:57:35.773 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:57:35.773 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:57:35.778 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.779+0000 7f790ab6f380 20 check_secure_mon_conn(): auth registy supported: methods=[2] modes=[2,1] 2021-11-11T20:57:35.778 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.779+0000 7f790ab6f380 20 check_secure_mon_conn(): mode 1 is insecure 2021-11-11T20:57:35.781 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.782+0000 7f78477fe700 20 reqs_thread_entry: start 2021-11-11T20:57:35.782 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.782+0000 7f790ab6f380 20 try_to_resolve_local_entities(): before: realm_id= realm_name= zonegroup_id= zonegroup_name= zone_id= zone_name=default 2021-11-11T20:57:35.782 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.782+0000 7f790ab6f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.783 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.783+0000 7f790ab6f380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:35.783 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.783+0000 7f790ab6f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.783 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.783+0000 7f790ab6f380 20 rados_obj.operate() r=0 bl.length=889 2021-11-11T20:57:35.793 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.793+0000 7f790ab6f380 20 RGWRados::pool_iterate: got zone_info.fda9d225-837d-46c3-b3f5-759647d0b924 2021-11-11T20:57:35.794 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.793+0000 7f790ab6f380 20 RGWRados::pool_iterate: got zonegroup_info.fd16e0e2-3d65-4610-ba66-92e4fc5ea00a 2021-11-11T20:57:35.794 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.793+0000 7f790ab6f380 20 RGWRados::pool_iterate: got zone_names.default 2021-11-11T20:57:35.794 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.793+0000 7f790ab6f380 20 RGWRados::pool_iterate: got zonegroups_names.default 2021-11-11T20:57:35.794 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.793+0000 7f790ab6f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.795 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.794+0000 7f790ab6f380 20 rados_obj.operate() r=0 bl.length=889 2021-11-11T20:57:35.795 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.794+0000 7f790ab6f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.795 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.794+0000 7f790ab6f380 20 rados_obj.operate() r=-2 bl.length=0 2021-11-11T20:57:35.796 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.794+0000 7f790ab6f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.796 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.794+0000 7f790ab6f380 20 rados_obj.operate() r=0 bl.length=46 2021-11-11T20:57:35.797 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.794+0000 7f790ab6f380 20 rados->read ofs=0 len=0 2021-11-11T20:57:35.797 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T20:57:35.795+0000 7f790ab6f380 20 rados_obj.operate() r=0 bl.length=362 2021-11-11T20:57:35.798 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T20:57:35.798 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "fda9d225-837d-46c3-b3f5-759647d0b924", 2021-11-11T20:57:35.799 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "default", 2021-11-11T20:57:35.799 INFO:teuthology.orchestra.run.smithi198.stdout: "domain_root": "default.rgw.meta:root", 2021-11-11T20:57:35.799 INFO:teuthology.orchestra.run.smithi198.stdout: "control_pool": "default.rgw.control", 2021-11-11T20:57:35.799 INFO:teuthology.orchestra.run.smithi198.stdout: "gc_pool": "default.rgw.log:gc", 2021-11-11T20:57:35.800 INFO:teuthology.orchestra.run.smithi198.stdout: "lc_pool": "default.rgw.log:lc", 2021-11-11T20:57:35.800 INFO:teuthology.orchestra.run.smithi198.stdout: "log_pool": "default.rgw.log", 2021-11-11T20:57:35.800 INFO:teuthology.orchestra.run.smithi198.stdout: "intent_log_pool": "default.rgw.log:intent", 2021-11-11T20:57:35.800 INFO:teuthology.orchestra.run.smithi198.stdout: "usage_log_pool": "default.rgw.log:usage", 2021-11-11T20:57:35.800 INFO:teuthology.orchestra.run.smithi198.stdout: "roles_pool": "default.rgw.meta:roles", 2021-11-11T20:57:35.801 INFO:teuthology.orchestra.run.smithi198.stdout: "reshard_pool": "default.rgw.log:reshard", 2021-11-11T20:57:35.801 INFO:teuthology.orchestra.run.smithi198.stdout: "user_keys_pool": "default.rgw.meta:users.keys", 2021-11-11T20:57:35.801 INFO:teuthology.orchestra.run.smithi198.stdout: "user_email_pool": "default.rgw.meta:users.email", 2021-11-11T20:57:35.801 INFO:teuthology.orchestra.run.smithi198.stdout: "user_swift_pool": "default.rgw.meta:users.swift", 2021-11-11T20:57:35.801 INFO:teuthology.orchestra.run.smithi198.stdout: "user_uid_pool": "default.rgw.meta:users.uid", 2021-11-11T20:57:35.802 INFO:teuthology.orchestra.run.smithi198.stdout: "otp_pool": "default.rgw.otp", 2021-11-11T20:57:35.802 INFO:teuthology.orchestra.run.smithi198.stdout: "system_key": { 2021-11-11T20:57:35.802 INFO:teuthology.orchestra.run.smithi198.stdout: "access_key": "", 2021-11-11T20:57:35.802 INFO:teuthology.orchestra.run.smithi198.stdout: "secret_key": "" 2021-11-11T20:57:35.803 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T20:57:35.803 INFO:teuthology.orchestra.run.smithi198.stdout: "placement_pools": [ 2021-11-11T20:57:35.803 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T20:57:35.803 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "default-placement", 2021-11-11T20:57:35.804 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T20:57:35.804 INFO:teuthology.orchestra.run.smithi198.stdout: "index_pool": "default.rgw.buckets.index", 2021-11-11T20:57:35.804 INFO:teuthology.orchestra.run.smithi198.stdout: "storage_classes": { 2021-11-11T20:57:35.804 INFO:teuthology.orchestra.run.smithi198.stdout: "STANDARD": { 2021-11-11T20:57:35.805 INFO:teuthology.orchestra.run.smithi198.stdout: "data_pool": "default.rgw.buckets.data", 2021-11-11T20:57:35.805 INFO:teuthology.orchestra.run.smithi198.stdout: "compression_type": "random" 2021-11-11T20:57:35.805 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T20:57:35.805 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T20:57:35.805 INFO:teuthology.orchestra.run.smithi198.stdout: "data_extra_pool": "default.rgw.buckets.non-ec", 2021-11-11T20:57:35.806 INFO:teuthology.orchestra.run.smithi198.stdout: "index_type": 0 2021-11-11T20:57:35.806 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T20:57:35.806 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T20:57:35.806 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T20:57:35.807 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "", 2021-11-11T20:57:35.807 INFO:teuthology.orchestra.run.smithi198.stdout: "notif_pool": "default.rgw.log:notif" 2021-11-11T20:57:35.807 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T20:57:35.808 DEBUG:tasks.util.rgw: json result: {'id': 'fda9d225-837d-46c3-b3f5-759647d0b924', 'name': 'default', 'domain_root': 'default.rgw.meta:root', 'control_pool': 'default.rgw.control', 'gc_pool': 'default.rgw.log:gc', 'lc_pool': 'default.rgw.log:lc', 'log_pool': 'default.rgw.log', 'intent_log_pool': 'default.rgw.log:intent', 'usage_log_pool': 'default.rgw.log:usage', 'roles_pool': 'default.rgw.meta:roles', 'reshard_pool': 'default.rgw.log:reshard', 'user_keys_pool': 'default.rgw.meta:users.keys', 'user_email_pool': 'default.rgw.meta:users.email', 'user_swift_pool': 'default.rgw.meta:users.swift', 'user_uid_pool': 'default.rgw.meta:users.uid', 'otp_pool': 'default.rgw.otp', 'system_key': {'access_key': '', 'secret_key': ''}, 'placement_pools': [{'key': 'default-placement', 'val': {'index_pool': 'default.rgw.buckets.index', 'storage_classes': {'STANDARD': {'data_pool': 'default.rgw.buckets.data', 'compression_type': 'random'}}, 'data_extra_pool': 'default.rgw.buckets.non-ec', 'index_type': 0}}], 'realm_id': '', 'notif_pool': 'default.rgw.log:notif'} 2021-11-11T20:57:35.809 INFO:tasks.rgw:Starting rgw... 2021-11-11T20:57:35.809 INFO:tasks.rgw:rgw c1.client.0 config is {'port': 8000} 2021-11-11T20:57:35.809 INFO:tasks.rgw:Using beast as radosgw frontend 2021-11-11T20:57:35.810 INFO:tasks.rgw.c1.client.0:Restarting daemon 2021-11-11T20:57:35.810 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c1 -k /etc/ceph/c1.client.0.keyring --log-file /var/log/ceph/rgw.c1.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.0.sock --foreground | sudo tee /var/log/ceph/rgw.c1.client.0.stdout 2>&1 2021-11-11T20:57:35.813 INFO:tasks.rgw.c1.client.0:Started 2021-11-11T20:57:35.814 INFO:tasks.rgw:rgw c1.client.1 config is {'port': 8001} 2021-11-11T20:57:35.814 INFO:tasks.rgw:Using beast as radosgw frontend 2021-11-11T20:57:35.815 INFO:tasks.rgw.c1.client.1:Restarting daemon 2021-11-11T20:57:35.815 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground | sudo tee /var/log/ceph/rgw.c1.client.1.stdout 2>&1 2021-11-11T20:57:35.817 INFO:tasks.rgw.c1.client.1:Started 2021-11-11T20:57:35.818 INFO:tasks.rgw:rgw c2.client.0 config is {'port': 8000} 2021-11-11T20:57:35.818 INFO:tasks.rgw:Using beast as radosgw frontend 2021-11-11T20:57:35.819 INFO:tasks.rgw.c2.client.0:Restarting daemon 2021-11-11T20:57:35.819 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground | sudo tee /var/log/ceph/rgw.c2.client.0.stdout 2>&1 2021-11-11T20:57:35.850 INFO:tasks.rgw.c2.client.0:Started 2021-11-11T20:57:35.851 INFO:tasks.rgw:rgw c2.client.1 config is {'port': 8001} 2021-11-11T20:57:35.851 INFO:tasks.rgw:Using beast as radosgw frontend 2021-11-11T20:57:35.852 INFO:tasks.rgw.c2.client.1:Restarting daemon 2021-11-11T20:57:35.852 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c2 -k /etc/ceph/c2.client.1.keyring --log-file /var/log/ceph/rgw.c2.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.1.sock --foreground | sudo tee /var/log/ceph/rgw.c2.client.1.stdout 2>&1 2021-11-11T20:57:35.854 INFO:tasks.rgw.c2.client.1:Started 2021-11-11T20:57:35.854 INFO:tasks.rgw:Polling c1.client.0 until it starts accepting connections on http://smithi096.front.sepia.ceph.com:8000/ 2021-11-11T20:57:35.855 DEBUG:teuthology.orchestra.run.smithi096:> curl http://smithi096.front.sepia.ceph.com:8000/ 2021-11-11T20:57:35.888 INFO:teuthology.orchestra.run.smithi096.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T20:57:35.889 INFO:teuthology.orchestra.run.smithi096.stderr: Dload Upload Total Spent Left Speed 2021-11-11T20:59:48.215 INFO:teuthology.orchestra.run.smithi096.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:30 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:32 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:34 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:36 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:38 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:40 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:42 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:44 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:46 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:48 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:50 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:52 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:54 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:56 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:58 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:00 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:30 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:32 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:34 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:36 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:38 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:40 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:42 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:44 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:46 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:48 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:50 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:52 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:54 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:56 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:58 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:00 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:10 --:--:-- 0curl: (7) Failed to connect to smithi096.front.sepia.ceph.com port 8000: Connection timed out 2021-11-11T20:59:48.217 DEBUG:teuthology.orchestra.run:got remote process result: 7 2021-11-11T20:59:49.219 DEBUG:teuthology.orchestra.run.smithi096:> curl http://smithi096.front.sepia.ceph.com:8000/ 2021-11-11T20:59:49.243 INFO:teuthology.orchestra.run.smithi096.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T20:59:49.243 INFO:teuthology.orchestra.run.smithi096.stderr: Dload Upload Total Spent Left Speed 2021-11-11T20:59:49.245 INFO:teuthology.orchestra.run.smithi096.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 71333 0 --:--:-- --:--:-- --:--:-- 71333 2021-11-11T20:59:49.247 INFO:teuthology.orchestra.run.smithi096.stdout:anonymous 2021-11-11T20:59:49.247 INFO:tasks.rgw:Polling c1.client.1 until it starts accepting connections on http://smithi096.front.sepia.ceph.com:8001/ 2021-11-11T20:59:49.248 DEBUG:teuthology.orchestra.run.smithi096:> curl http://smithi096.front.sepia.ceph.com:8001/ 2021-11-11T20:59:49.307 INFO:teuthology.orchestra.run.smithi096.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T20:59:49.308 INFO:teuthology.orchestra.run.smithi096.stderr: Dload Upload Total Spent Left Speed 2021-11-11T20:59:49.309 INFO:teuthology.orchestra.run.smithi096.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 104k 0 --:--:-- --:--:-- --:--:-- 104k 2021-11-11T20:59:49.311 INFO:teuthology.orchestra.run.smithi096.stdout:anonymous 2021-11-11T20:59:49.311 INFO:tasks.rgw:Polling c2.client.0 until it starts accepting connections on http://smithi198.front.sepia.ceph.com:8000/ 2021-11-11T20:59:49.311 DEBUG:teuthology.orchestra.run.smithi198:> curl http://smithi198.front.sepia.ceph.com:8000/ 2021-11-11T20:59:49.352 INFO:teuthology.orchestra.run.smithi198.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T20:59:49.352 INFO:teuthology.orchestra.run.smithi198.stderr: Dload Upload Total Spent Left Speed 2021-11-11T20:59:49.355 INFO:teuthology.orchestra.run.smithi198.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 71333 0 --:--:-- --:--:-- --:--:-- 71333 2021-11-11T20:59:49.356 INFO:teuthology.orchestra.run.smithi198.stdout:anonymous 2021-11-11T20:59:49.356 INFO:tasks.rgw:Polling c2.client.1 until it starts accepting connections on http://smithi198.front.sepia.ceph.com:8001/ 2021-11-11T20:59:49.357 DEBUG:teuthology.orchestra.run.smithi198:> curl http://smithi198.front.sepia.ceph.com:8001/ 2021-11-11T20:59:49.416 INFO:teuthology.orchestra.run.smithi198.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T20:59:49.417 INFO:teuthology.orchestra.run.smithi198.stderr: Dload Upload Total Spent Left Speed 2021-11-11T20:59:49.418 INFO:teuthology.orchestra.run.smithi198.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 104k 0 --:--:-- --:--:-- --:--:-- 104k 2021-11-11T20:59:49.419 INFO:teuthology.orchestra.run.smithi198.stdout:anonymous 2021-11-11T20:59:49.420 INFO:teuthology.run_tasks:Running task rgw-multisite... 2021-11-11T20:59:49.628 INFO:tasks.rgw_multisite:creating realm.. 2021-11-11T20:59:49.629 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['realm', 'create', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T20:59:49.630 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'realm', 'create', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T20:59:49.630 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 realm create --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T20:59:49.675 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:59:49.675 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:59:49.717 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T20:59:49.718 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T20:59:49.718 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-realm", 2021-11-11T20:59:49.718 INFO:teuthology.orchestra.run.smithi096.stdout: "current_period": "ab45263b-38f8-4e1a-9f37-1e83128a94ad", 2021-11-11T20:59:49.718 INFO:teuthology.orchestra.run.smithi096.stdout: "epoch": 1 2021-11-11T20:59:49.719 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T20:59:49.720 INFO:tasks.rgw_multisite:creating master zonegroup.. 2021-11-11T20:59:49.721 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'set', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T20:59:49.721 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'set', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T20:59:49.722 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup set --rgw-zonegroup test-zonegroup --rgw-realm test-realm --default --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T20:59:49.801 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:59:49.802 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:59:49.814 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:59:49.814+0000 7f7fcb252380 -1 old format 2021-11-11T20:59:49.823 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T20:59:49.823 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "test-zonegroup", 2021-11-11T20:59:49.823 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zonegroup", 2021-11-11T20:59:49.823 INFO:teuthology.orchestra.run.smithi096.stdout: "api_name": "", 2021-11-11T20:59:49.824 INFO:teuthology.orchestra.run.smithi096.stdout: "is_master": "true", 2021-11-11T20:59:49.824 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T20:59:49.824 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T20:59:49.824 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T20:59:49.825 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames": [], 2021-11-11T20:59:49.825 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames_s3website": [], 2021-11-11T20:59:49.825 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zone": "", 2021-11-11T20:59:49.825 INFO:teuthology.orchestra.run.smithi096.stdout: "zones": [], 2021-11-11T20:59:49.826 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_targets": [], 2021-11-11T20:59:49.826 INFO:teuthology.orchestra.run.smithi096.stdout: "default_placement": "", 2021-11-11T20:59:49.826 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T20:59:49.826 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_policy": { 2021-11-11T20:59:49.826 INFO:teuthology.orchestra.run.smithi096.stdout: "groups": [] 2021-11-11T20:59:49.827 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:49.827 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T20:59:49.828 INFO:tasks.rgw_multisite:creating master zone.. 2021-11-11T20:59:49.829 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'create', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--master', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--endpoints', 'http://smithi096.front.sepia.ceph.com:8000', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T20:59:49.829 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'create', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--master', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--endpoints', 'http://smithi096.front.sepia.ceph.com:8000', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T20:59:49.829 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone create --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --default --master --access-key UBE0V2NN5ZCDGTQQ --secret H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ --endpoints http://smithi096.front.sepia.ceph.com:8000 --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T20:59:49.916 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:59:49.916 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:59:49.951 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T20:59:49.952 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T20:59:49.952 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone1", 2021-11-11T20:59:49.952 INFO:teuthology.orchestra.run.smithi096.stdout: "domain_root": "test-zone1.rgw.meta:root", 2021-11-11T20:59:49.952 INFO:teuthology.orchestra.run.smithi096.stdout: "control_pool": "test-zone1.rgw.control", 2021-11-11T20:59:49.953 INFO:teuthology.orchestra.run.smithi096.stdout: "gc_pool": "test-zone1.rgw.log:gc", 2021-11-11T20:59:49.953 INFO:teuthology.orchestra.run.smithi096.stdout: "lc_pool": "test-zone1.rgw.log:lc", 2021-11-11T20:59:49.953 INFO:teuthology.orchestra.run.smithi096.stdout: "log_pool": "test-zone1.rgw.log", 2021-11-11T20:59:49.953 INFO:teuthology.orchestra.run.smithi096.stdout: "intent_log_pool": "test-zone1.rgw.log:intent", 2021-11-11T20:59:49.954 INFO:teuthology.orchestra.run.smithi096.stdout: "usage_log_pool": "test-zone1.rgw.log:usage", 2021-11-11T20:59:49.954 INFO:teuthology.orchestra.run.smithi096.stdout: "roles_pool": "test-zone1.rgw.meta:roles", 2021-11-11T20:59:49.955 INFO:teuthology.orchestra.run.smithi096.stdout: "reshard_pool": "test-zone1.rgw.log:reshard", 2021-11-11T20:59:49.955 INFO:teuthology.orchestra.run.smithi096.stdout: "user_keys_pool": "test-zone1.rgw.meta:users.keys", 2021-11-11T20:59:49.956 INFO:teuthology.orchestra.run.smithi096.stdout: "user_email_pool": "test-zone1.rgw.meta:users.email", 2021-11-11T20:59:49.956 INFO:teuthology.orchestra.run.smithi096.stdout: "user_swift_pool": "test-zone1.rgw.meta:users.swift", 2021-11-11T20:59:49.956 INFO:teuthology.orchestra.run.smithi096.stdout: "user_uid_pool": "test-zone1.rgw.meta:users.uid", 2021-11-11T20:59:49.956 INFO:teuthology.orchestra.run.smithi096.stdout: "otp_pool": "test-zone1.rgw.otp", 2021-11-11T20:59:49.956 INFO:teuthology.orchestra.run.smithi096.stdout: "system_key": { 2021-11-11T20:59:49.957 INFO:teuthology.orchestra.run.smithi096.stdout: "access_key": "UBE0V2NN5ZCDGTQQ", 2021-11-11T20:59:49.957 INFO:teuthology.orchestra.run.smithi096.stdout: "secret_key": "H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ" 2021-11-11T20:59:49.957 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T20:59:49.957 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_pools": [ 2021-11-11T20:59:49.958 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T20:59:49.958 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "default-placement", 2021-11-11T20:59:49.958 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T20:59:49.958 INFO:teuthology.orchestra.run.smithi096.stdout: "index_pool": "test-zone1.rgw.buckets.index", 2021-11-11T20:59:49.958 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": { 2021-11-11T20:59:49.959 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD": { 2021-11-11T20:59:49.959 INFO:teuthology.orchestra.run.smithi096.stdout: "data_pool": "test-zone1.rgw.buckets.data" 2021-11-11T20:59:49.959 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:49.959 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T20:59:49.960 INFO:teuthology.orchestra.run.smithi096.stdout: "data_extra_pool": "test-zone1.rgw.buckets.non-ec", 2021-11-11T20:59:49.960 INFO:teuthology.orchestra.run.smithi096.stdout: "index_type": 0 2021-11-11T20:59:49.960 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:49.960 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:49.960 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T20:59:49.961 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T20:59:49.961 INFO:teuthology.orchestra.run.smithi096.stdout: "notif_pool": "test-zone1.rgw.log:notif" 2021-11-11T20:59:49.961 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T20:59:49.963 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool create test-zone1.rgw.buckets.data 64 64 --cluster c1 2021-11-11T20:59:50.524 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'test-zone1.rgw.buckets.data' created 2021-11-11T20:59:50.541 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool application enable test-zone1.rgw.buckets.data rgw --cluster c1 2021-11-11T20:59:51.541 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rgw' on pool 'test-zone1.rgw.buckets.data' 2021-11-11T20:59:51.559 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'placement', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T20:59:51.559 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'placement', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T20:59:51.560 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone placement --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T20:59:51.610 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:59:51.611 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:59:51.639 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T20:59:51.639 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T20:59:51.640 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone1", 2021-11-11T20:59:51.640 INFO:teuthology.orchestra.run.smithi096.stdout: "domain_root": "test-zone1.rgw.meta:root", 2021-11-11T20:59:51.640 INFO:teuthology.orchestra.run.smithi096.stdout: "control_pool": "test-zone1.rgw.control", 2021-11-11T20:59:51.640 INFO:teuthology.orchestra.run.smithi096.stdout: "gc_pool": "test-zone1.rgw.log:gc", 2021-11-11T20:59:51.640 INFO:teuthology.orchestra.run.smithi096.stdout: "lc_pool": "test-zone1.rgw.log:lc", 2021-11-11T20:59:51.641 INFO:teuthology.orchestra.run.smithi096.stdout: "log_pool": "test-zone1.rgw.log", 2021-11-11T20:59:51.641 INFO:teuthology.orchestra.run.smithi096.stdout: "intent_log_pool": "test-zone1.rgw.log:intent", 2021-11-11T20:59:51.641 INFO:teuthology.orchestra.run.smithi096.stdout: "usage_log_pool": "test-zone1.rgw.log:usage", 2021-11-11T20:59:51.641 INFO:teuthology.orchestra.run.smithi096.stdout: "roles_pool": "test-zone1.rgw.meta:roles", 2021-11-11T20:59:51.642 INFO:teuthology.orchestra.run.smithi096.stdout: "reshard_pool": "test-zone1.rgw.log:reshard", 2021-11-11T20:59:51.642 INFO:teuthology.orchestra.run.smithi096.stdout: "user_keys_pool": "test-zone1.rgw.meta:users.keys", 2021-11-11T20:59:51.642 INFO:teuthology.orchestra.run.smithi096.stdout: "user_email_pool": "test-zone1.rgw.meta:users.email", 2021-11-11T20:59:51.642 INFO:teuthology.orchestra.run.smithi096.stdout: "user_swift_pool": "test-zone1.rgw.meta:users.swift", 2021-11-11T20:59:51.642 INFO:teuthology.orchestra.run.smithi096.stdout: "user_uid_pool": "test-zone1.rgw.meta:users.uid", 2021-11-11T20:59:51.643 INFO:teuthology.orchestra.run.smithi096.stdout: "otp_pool": "test-zone1.rgw.otp", 2021-11-11T20:59:51.643 INFO:teuthology.orchestra.run.smithi096.stdout: "system_key": { 2021-11-11T20:59:51.643 INFO:teuthology.orchestra.run.smithi096.stdout: "access_key": "UBE0V2NN5ZCDGTQQ", 2021-11-11T20:59:51.643 INFO:teuthology.orchestra.run.smithi096.stdout: "secret_key": "H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ" 2021-11-11T20:59:51.644 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T20:59:51.644 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_pools": [ 2021-11-11T20:59:51.644 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T20:59:51.644 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "default-placement", 2021-11-11T20:59:51.644 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T20:59:51.645 INFO:teuthology.orchestra.run.smithi096.stdout: "index_pool": "test-zone1.rgw.buckets.index", 2021-11-11T20:59:51.645 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": { 2021-11-11T20:59:51.645 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD": { 2021-11-11T20:59:51.645 INFO:teuthology.orchestra.run.smithi096.stdout: "data_pool": "test-zone1.rgw.buckets.data", 2021-11-11T20:59:51.645 INFO:teuthology.orchestra.run.smithi096.stdout: "compression_type": "random" 2021-11-11T20:59:51.646 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:51.646 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T20:59:51.646 INFO:teuthology.orchestra.run.smithi096.stdout: "data_extra_pool": "test-zone1.rgw.buckets.non-ec", 2021-11-11T20:59:51.646 INFO:teuthology.orchestra.run.smithi096.stdout: "index_type": 0 2021-11-11T20:59:51.647 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:51.647 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:51.647 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T20:59:51.647 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "", 2021-11-11T20:59:51.648 INFO:teuthology.orchestra.run.smithi096.stdout: "notif_pool": "test-zone1.rgw.log:notif" 2021-11-11T20:59:51.648 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T20:59:51.649 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T20:59:51.650 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T20:59:51.650 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T20:59:51.691 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T20:59:51.691 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T20:59:51.701 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T20:59:51.701+0000 7f7a3ec1d380 0 period (ab45263b-38f8-4e1a-9f37-1e83128a94ad does not have zone badb2b8d-7821-4be0-b9ef-60c04d880ece configured 2021-11-11T20:59:59.636 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T20:59:59.637 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T20:59:59.637 INFO:teuthology.orchestra.run.smithi096.stdout: "epoch": 1, 2021-11-11T20:59:59.637 INFO:teuthology.orchestra.run.smithi096.stdout: "predecessor_uuid": "ab45263b-38f8-4e1a-9f37-1e83128a94ad", 2021-11-11T20:59:59.637 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": [], 2021-11-11T20:59:59.638 INFO:teuthology.orchestra.run.smithi096.stdout: "period_map": { 2021-11-11T20:59:59.638 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T20:59:59.638 INFO:teuthology.orchestra.run.smithi096.stdout: "zonegroups": [ 2021-11-11T20:59:59.638 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T20:59:59.639 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "test-zonegroup", 2021-11-11T20:59:59.639 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zonegroup", 2021-11-11T20:59:59.639 INFO:teuthology.orchestra.run.smithi096.stdout: "api_name": "", 2021-11-11T20:59:59.639 INFO:teuthology.orchestra.run.smithi096.stdout: "is_master": "true", 2021-11-11T20:59:59.639 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T20:59:59.640 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T20:59:59.640 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T20:59:59.640 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames": [], 2021-11-11T20:59:59.640 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames_s3website": [], 2021-11-11T20:59:59.641 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T20:59:59.641 INFO:teuthology.orchestra.run.smithi096.stdout: "zones": [ 2021-11-11T20:59:59.641 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T20:59:59.641 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T20:59:59.641 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone1", 2021-11-11T20:59:59.642 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T20:59:59.642 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T20:59:59.642 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T20:59:59.642 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T20:59:59.642 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "false", 2021-11-11T20:59:59.643 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T20:59:59.643 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T20:59:59.643 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "", 2021-11-11T20:59:59.643 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "true", 2021-11-11T20:59:59.643 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [], 2021-11-11T20:59:59.644 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T20:59:59.644 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:59.644 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T20:59:59.644 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_targets": [ 2021-11-11T20:59:59.644 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T20:59:59.645 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "default-placement", 2021-11-11T20:59:59.645 INFO:teuthology.orchestra.run.smithi096.stdout: "tags": [], 2021-11-11T20:59:59.645 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": [ 2021-11-11T20:59:59.646 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD" 2021-11-11T20:59:59.646 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T20:59:59.646 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:59.646 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T20:59:59.646 INFO:teuthology.orchestra.run.smithi096.stdout: "default_placement": "default-placement", 2021-11-11T20:59:59.647 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T20:59:59.647 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_policy": { 2021-11-11T20:59:59.647 INFO:teuthology.orchestra.run.smithi096.stdout: "groups": [] 2021-11-11T20:59:59.647 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:59.647 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:59.648 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T20:59:59.648 INFO:teuthology.orchestra.run.smithi096.stdout: "short_zone_ids": [ 2021-11-11T20:59:59.648 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T20:59:59.648 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T20:59:59.648 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 1844113099 2021-11-11T20:59:59.649 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:59.649 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T20:59:59.649 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T20:59:59.649 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zonegroup": "test-zonegroup", 2021-11-11T20:59:59.650 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T20:59:59.650 INFO:teuthology.orchestra.run.smithi096.stdout: "period_config": { 2021-11-11T20:59:59.650 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_quota": { 2021-11-11T20:59:59.650 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T20:59:59.651 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T20:59:59.651 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T20:59:59.651 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T20:59:59.651 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T20:59:59.651 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T20:59:59.652 INFO:teuthology.orchestra.run.smithi096.stdout: "user_quota": { 2021-11-11T20:59:59.652 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T20:59:59.652 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T20:59:59.652 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T20:59:59.653 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T20:59:59.653 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T20:59:59.653 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T20:59:59.653 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T20:59:59.653 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T20:59:59.654 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_name": "test-realm", 2021-11-11T20:59:59.654 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T20:59:59.654 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T20:59:59.655 DEBUG:tasks.rgw.c1.client.0:waiting for process to exit 2021-11-11T20:59:59.656 INFO:teuthology.orchestra.run:waiting for 300 2021-11-11T20:59:59.696 INFO:tasks.rgw.c1.client.0.smithi096.stdout:2021-11-11T20:59:59.697+0000 7f6079055700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c1 -k /etc/ceph/c1.client.0.keyring --log-file /var/log/ceph/rgw.c1.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.0.sock --foreground (PID: 37280) UID: 0 2021-11-11T20:59:59.697 INFO:tasks.rgw.c1.client.0.smithi096.stdout:2021-11-11T20:59:59.697+0000 7f6089fa1440 -1 rgw dbstore: shutting down 2021-11-11T21:00:03.977 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.0 is failed for ~0s 2021-11-11T21:00:05.758 INFO:tasks.rgw.c1.client.0:Stopped 2021-11-11T21:00:05.758 INFO:tasks.rgw.c1.client.0:Restarting daemon 2021-11-11T21:00:05.759 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c1 -k /etc/ceph/c1.client.0.keyring --log-file /var/log/ceph/rgw.c1.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.0.sock --foreground --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c1.client.0.stdout 2>&1 2021-11-11T21:00:05.763 INFO:tasks.rgw.c1.client.0:Started 2021-11-11T21:00:05.763 DEBUG:teuthology.orchestra.run.smithi096:> curl http://smithi096.front.sepia.ceph.com:8000 2021-11-11T21:00:05.784 INFO:teuthology.orchestra.run.smithi096.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T21:00:05.785 INFO:teuthology.orchestra.run.smithi096.stderr: Dload Upload Total Spent Left Speed 2021-11-11T21:00:09.685 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.0 has been restored 2021-11-11T21:02:16.130 DEBUG:teuthology.orchestra.run:got remote process result: 7 2021-11-11T21:02:16.132 INFO:teuthology.orchestra.run.smithi096.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:30 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:32 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:34 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:36 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:38 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:40 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:42 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:44 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:46 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:48 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:50 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:52 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:54 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:56 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:58 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:00 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:30 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:32 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:34 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:36 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:38 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:40 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:42 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:44 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:46 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:48 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:50 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:52 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:54 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:56 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:58 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:00 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:08 --:--:-- 0curl: (7) Failed to connect to smithi096.front.sepia.ceph.com port 8000: Connection timed out 2021-11-11T21:02:17.133 DEBUG:teuthology.orchestra.run.smithi096:> curl http://smithi096.front.sepia.ceph.com:8000 2021-11-11T21:02:17.157 INFO:teuthology.orchestra.run.smithi096.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T21:02:17.157 INFO:teuthology.orchestra.run.smithi096.stderr: Dload Upload Total Spent Left Speed 2021-11-11T21:02:17.159 INFO:teuthology.orchestra.run.smithi096.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 104k 0 --:--:-- --:--:-- --:--:-- 104k 2021-11-11T21:02:17.160 INFO:teuthology.orchestra.run.smithi096.stdout:anonymous 2021-11-11T21:02:17.161 INFO:tasks.rgw_multisite:creating admin user.. 2021-11-11T21:02:17.161 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'create', '--uid', 'realm-admin', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--display-name', 'Realm Admin', '--system', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:02:17.162 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'create', '--uid', 'realm-admin', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--display-name', 'Realm Admin', '--system', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:02:17.162 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user create --uid realm-admin --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --display-name 'Realm Admin' --system --access-key UBE0V2NN5ZCDGTQQ --secret H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:02:17.255 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:02:17.256 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:02:17.319 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:02:17.320 INFO:teuthology.orchestra.run.smithi096.stdout: "user_id": "realm-admin", 2021-11-11T21:02:17.320 INFO:teuthology.orchestra.run.smithi096.stdout: "display_name": "Realm Admin", 2021-11-11T21:02:17.320 INFO:teuthology.orchestra.run.smithi096.stdout: "email": "", 2021-11-11T21:02:17.320 INFO:teuthology.orchestra.run.smithi096.stdout: "suspended": 0, 2021-11-11T21:02:17.321 INFO:teuthology.orchestra.run.smithi096.stdout: "max_buckets": 1000, 2021-11-11T21:02:17.321 INFO:teuthology.orchestra.run.smithi096.stdout: "subusers": [], 2021-11-11T21:02:17.321 INFO:teuthology.orchestra.run.smithi096.stdout: "keys": [ 2021-11-11T21:02:17.321 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:02:17.321 INFO:teuthology.orchestra.run.smithi096.stdout: "user": "realm-admin", 2021-11-11T21:02:17.322 INFO:teuthology.orchestra.run.smithi096.stdout: "access_key": "UBE0V2NN5ZCDGTQQ", 2021-11-11T21:02:17.322 INFO:teuthology.orchestra.run.smithi096.stdout: "secret_key": "H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ" 2021-11-11T21:02:17.322 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:02:17.322 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:02:17.323 INFO:teuthology.orchestra.run.smithi096.stdout: "swift_keys": [], 2021-11-11T21:02:17.323 INFO:teuthology.orchestra.run.smithi096.stdout: "caps": [], 2021-11-11T21:02:17.323 INFO:teuthology.orchestra.run.smithi096.stdout: "op_mask": "read, write, delete", 2021-11-11T21:02:17.323 INFO:teuthology.orchestra.run.smithi096.stdout: "system": "true", 2021-11-11T21:02:17.323 INFO:teuthology.orchestra.run.smithi096.stdout: "default_placement": "", 2021-11-11T21:02:17.324 INFO:teuthology.orchestra.run.smithi096.stdout: "default_storage_class": "", 2021-11-11T21:02:17.324 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_tags": [], 2021-11-11T21:02:17.324 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_quota": { 2021-11-11T21:02:17.324 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T21:02:17.324 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T21:02:17.325 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T21:02:17.325 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T21:02:17.325 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T21:02:17.325 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:02:17.326 INFO:teuthology.orchestra.run.smithi096.stdout: "user_quota": { 2021-11-11T21:02:17.326 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T21:02:17.326 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T21:02:17.326 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T21:02:17.327 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T21:02:17.327 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T21:02:17.327 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:02:17.327 INFO:teuthology.orchestra.run.smithi096.stdout: "temp_url_keys": [], 2021-11-11T21:02:17.327 INFO:teuthology.orchestra.run.smithi096.stdout: "type": "rgw", 2021-11-11T21:02:17.328 INFO:teuthology.orchestra.run.smithi096.stdout: "mfa_ids": [] 2021-11-11T21:02:17.328 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:02:17.328 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T21:02:17.341 INFO:tasks.rgw_multisite:pulling realm configuration to c2 2021-11-11T21:02:17.342 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['realm', 'pull', '--rgw-realm', 'test-realm', '--url', 'http://smithi096.front.sepia.ceph.com:8000', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:02:17.342 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'realm', 'pull', '--rgw-realm', 'test-realm', '--url', 'http://smithi096.front.sepia.ceph.com:8000', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:02:17.342 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 realm pull --rgw-realm test-realm --url http://smithi096.front.sepia.ceph.com:8000 --access-key UBE0V2NN5ZCDGTQQ --secret H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ --cluster c2 --debug-rgw 0 --debug-ms 0 2021-11-11T21:02:17.395 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:02:17.395 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:02:17.487 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:02:17.487 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:02:17.488 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-realm", 2021-11-11T21:02:17.488 INFO:teuthology.orchestra.run.smithi198.stdout: "current_period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:02:17.488 INFO:teuthology.orchestra.run.smithi198.stdout: "epoch": 2 2021-11-11T21:02:17.488 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:02:17.490 INFO:tasks.rgw_multisite:creating zone.. 2021-11-11T21:02:17.490 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'create', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--endpoints', 'http://smithi198.front.sepia.ceph.com:8000', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:02:17.491 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'create', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--default', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--endpoints', 'http://smithi198.front.sepia.ceph.com:8000', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:02:17.491 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone create --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --default --access-key UBE0V2NN5ZCDGTQQ --secret H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ --endpoints http://smithi198.front.sepia.ceph.com:8000 --cluster c2 --debug-rgw 0 --debug-ms 0 2021-11-11T21:02:17.538 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:02:17.539 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:02:17.567 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:02:17.568+0000 7f3817368380 0 failed reading obj info from .rgw.root:zone_info.badb2b8d-7821-4be0-b9ef-60c04d880ece: (2) No such file or directory 2021-11-11T21:02:17.568 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:02:17.568+0000 7f3817368380 0 WARNING: could not read zone params for zone id=badb2b8d-7821-4be0-b9ef-60c04d880ece name=test-zone1 2021-11-11T21:02:17.575 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:02:17.575 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:02:17.575 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone2", 2021-11-11T21:02:17.575 INFO:teuthology.orchestra.run.smithi198.stdout: "domain_root": "test-zone2.rgw.meta:root", 2021-11-11T21:02:17.575 INFO:teuthology.orchestra.run.smithi198.stdout: "control_pool": "test-zone2.rgw.control", 2021-11-11T21:02:17.576 INFO:teuthology.orchestra.run.smithi198.stdout: "gc_pool": "test-zone2.rgw.log:gc", 2021-11-11T21:02:17.576 INFO:teuthology.orchestra.run.smithi198.stdout: "lc_pool": "test-zone2.rgw.log:lc", 2021-11-11T21:02:17.576 INFO:teuthology.orchestra.run.smithi198.stdout: "log_pool": "test-zone2.rgw.log", 2021-11-11T21:02:17.576 INFO:teuthology.orchestra.run.smithi198.stdout: "intent_log_pool": "test-zone2.rgw.log:intent", 2021-11-11T21:02:17.576 INFO:teuthology.orchestra.run.smithi198.stdout: "usage_log_pool": "test-zone2.rgw.log:usage", 2021-11-11T21:02:17.577 INFO:teuthology.orchestra.run.smithi198.stdout: "roles_pool": "test-zone2.rgw.meta:roles", 2021-11-11T21:02:17.577 INFO:teuthology.orchestra.run.smithi198.stdout: "reshard_pool": "test-zone2.rgw.log:reshard", 2021-11-11T21:02:17.577 INFO:teuthology.orchestra.run.smithi198.stdout: "user_keys_pool": "test-zone2.rgw.meta:users.keys", 2021-11-11T21:02:17.577 INFO:teuthology.orchestra.run.smithi198.stdout: "user_email_pool": "test-zone2.rgw.meta:users.email", 2021-11-11T21:02:17.577 INFO:teuthology.orchestra.run.smithi198.stdout: "user_swift_pool": "test-zone2.rgw.meta:users.swift", 2021-11-11T21:02:17.578 INFO:teuthology.orchestra.run.smithi198.stdout: "user_uid_pool": "test-zone2.rgw.meta:users.uid", 2021-11-11T21:02:17.578 INFO:teuthology.orchestra.run.smithi198.stdout: "otp_pool": "test-zone2.rgw.otp", 2021-11-11T21:02:17.578 INFO:teuthology.orchestra.run.smithi198.stdout: "system_key": { 2021-11-11T21:02:17.578 INFO:teuthology.orchestra.run.smithi198.stdout: "access_key": "UBE0V2NN5ZCDGTQQ", 2021-11-11T21:02:17.579 INFO:teuthology.orchestra.run.smithi198.stdout: "secret_key": "H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ" 2021-11-11T21:02:17.579 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:02:17.579 INFO:teuthology.orchestra.run.smithi198.stdout: "placement_pools": [ 2021-11-11T21:02:17.579 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:02:17.579 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "default-placement", 2021-11-11T21:02:17.580 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:02:17.580 INFO:teuthology.orchestra.run.smithi198.stdout: "index_pool": "test-zone2.rgw.buckets.index", 2021-11-11T21:02:17.580 INFO:teuthology.orchestra.run.smithi198.stdout: "storage_classes": { 2021-11-11T21:02:17.580 INFO:teuthology.orchestra.run.smithi198.stdout: "STANDARD": { 2021-11-11T21:02:17.580 INFO:teuthology.orchestra.run.smithi198.stdout: "data_pool": "test-zone2.rgw.buckets.data" 2021-11-11T21:02:17.581 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:17.581 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:02:17.581 INFO:teuthology.orchestra.run.smithi198.stdout: "data_extra_pool": "test-zone2.rgw.buckets.non-ec", 2021-11-11T21:02:17.581 INFO:teuthology.orchestra.run.smithi198.stdout: "index_type": 0 2021-11-11T21:02:17.581 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:17.582 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:17.582 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:02:17.582 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:02:17.582 INFO:teuthology.orchestra.run.smithi198.stdout: "notif_pool": "test-zone2.rgw.log:notif" 2021-11-11T21:02:17.582 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:02:17.584 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool create test-zone2.rgw.buckets.data 64 64 --cluster c2 2021-11-11T21:02:18.619 INFO:teuthology.orchestra.run.smithi198.stderr:pool 'test-zone2.rgw.buckets.data' created 2021-11-11T21:02:18.636 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool application enable test-zone2.rgw.buckets.data rgw --cluster c2 2021-11-11T21:02:19.645 INFO:teuthology.orchestra.run.smithi198.stderr:enabled application 'rgw' on pool 'test-zone2.rgw.buckets.data' 2021-11-11T21:02:19.664 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'placement', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:02:19.664 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'placement', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:02:19.665 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone placement --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c2 --debug-rgw 0 --debug-ms 0 2021-11-11T21:02:19.709 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:02:19.710 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:02:19.737 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:02:19.737 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:02:19.738 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone2", 2021-11-11T21:02:19.738 INFO:teuthology.orchestra.run.smithi198.stdout: "domain_root": "test-zone2.rgw.meta:root", 2021-11-11T21:02:19.738 INFO:teuthology.orchestra.run.smithi198.stdout: "control_pool": "test-zone2.rgw.control", 2021-11-11T21:02:19.738 INFO:teuthology.orchestra.run.smithi198.stdout: "gc_pool": "test-zone2.rgw.log:gc", 2021-11-11T21:02:19.739 INFO:teuthology.orchestra.run.smithi198.stdout: "lc_pool": "test-zone2.rgw.log:lc", 2021-11-11T21:02:19.739 INFO:teuthology.orchestra.run.smithi198.stdout: "log_pool": "test-zone2.rgw.log", 2021-11-11T21:02:19.739 INFO:teuthology.orchestra.run.smithi198.stdout: "intent_log_pool": "test-zone2.rgw.log:intent", 2021-11-11T21:02:19.739 INFO:teuthology.orchestra.run.smithi198.stdout: "usage_log_pool": "test-zone2.rgw.log:usage", 2021-11-11T21:02:19.740 INFO:teuthology.orchestra.run.smithi198.stdout: "roles_pool": "test-zone2.rgw.meta:roles", 2021-11-11T21:02:19.740 INFO:teuthology.orchestra.run.smithi198.stdout: "reshard_pool": "test-zone2.rgw.log:reshard", 2021-11-11T21:02:19.740 INFO:teuthology.orchestra.run.smithi198.stdout: "user_keys_pool": "test-zone2.rgw.meta:users.keys", 2021-11-11T21:02:19.740 INFO:teuthology.orchestra.run.smithi198.stdout: "user_email_pool": "test-zone2.rgw.meta:users.email", 2021-11-11T21:02:19.740 INFO:teuthology.orchestra.run.smithi198.stdout: "user_swift_pool": "test-zone2.rgw.meta:users.swift", 2021-11-11T21:02:19.741 INFO:teuthology.orchestra.run.smithi198.stdout: "user_uid_pool": "test-zone2.rgw.meta:users.uid", 2021-11-11T21:02:19.741 INFO:teuthology.orchestra.run.smithi198.stdout: "otp_pool": "test-zone2.rgw.otp", 2021-11-11T21:02:19.741 INFO:teuthology.orchestra.run.smithi198.stdout: "system_key": { 2021-11-11T21:02:19.741 INFO:teuthology.orchestra.run.smithi198.stdout: "access_key": "UBE0V2NN5ZCDGTQQ", 2021-11-11T21:02:19.742 INFO:teuthology.orchestra.run.smithi198.stdout: "secret_key": "H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ" 2021-11-11T21:02:19.742 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:02:19.742 INFO:teuthology.orchestra.run.smithi198.stdout: "placement_pools": [ 2021-11-11T21:02:19.742 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:02:19.742 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "default-placement", 2021-11-11T21:02:19.743 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:02:19.743 INFO:teuthology.orchestra.run.smithi198.stdout: "index_pool": "test-zone2.rgw.buckets.index", 2021-11-11T21:02:19.743 INFO:teuthology.orchestra.run.smithi198.stdout: "storage_classes": { 2021-11-11T21:02:19.743 INFO:teuthology.orchestra.run.smithi198.stdout: "STANDARD": { 2021-11-11T21:02:19.743 INFO:teuthology.orchestra.run.smithi198.stdout: "data_pool": "test-zone2.rgw.buckets.data", 2021-11-11T21:02:19.744 INFO:teuthology.orchestra.run.smithi198.stdout: "compression_type": "random" 2021-11-11T21:02:19.744 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:19.744 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:02:19.744 INFO:teuthology.orchestra.run.smithi198.stdout: "data_extra_pool": "test-zone2.rgw.buckets.non-ec", 2021-11-11T21:02:19.745 INFO:teuthology.orchestra.run.smithi198.stdout: "index_type": 0 2021-11-11T21:02:19.745 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:19.745 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:19.745 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:02:19.745 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "", 2021-11-11T21:02:19.746 INFO:teuthology.orchestra.run.smithi198.stdout: "notif_pool": "test-zone2.rgw.log:notif" 2021-11-11T21:02:19.746 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:02:19.747 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:02:19.748 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:02:19.748 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2021-11-11T21:02:19.789 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:02:19.789 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:02:19.799 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:02:19.799+0000 7feb8429b380 0 period (2aa9d15b-21de-40bc-a7d3-3d5e5933111b does not have zone 82b05a71-8273-4ec7-b7c6-953c0f40d3e0 configured 2021-11-11T21:02:27.684 INFO:teuthology.orchestra.run.smithi198.stderr:Sending period to new master zone badb2b8d-7821-4be0-b9ef-60c04d880ece 2021-11-11T21:02:27.736 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:02:27.736 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:02:27.736 INFO:teuthology.orchestra.run.smithi198.stdout: "epoch": 2, 2021-11-11T21:02:27.737 INFO:teuthology.orchestra.run.smithi198.stdout: "predecessor_uuid": "ab45263b-38f8-4e1a-9f37-1e83128a94ad", 2021-11-11T21:02:27.737 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": [], 2021-11-11T21:02:27.737 INFO:teuthology.orchestra.run.smithi198.stdout: "period_map": { 2021-11-11T21:02:27.737 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:02:27.738 INFO:teuthology.orchestra.run.smithi198.stdout: "zonegroups": [ 2021-11-11T21:02:27.738 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:02:27.738 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "test-zonegroup", 2021-11-11T21:02:27.738 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zonegroup", 2021-11-11T21:02:27.738 INFO:teuthology.orchestra.run.smithi198.stdout: "api_name": "", 2021-11-11T21:02:27.739 INFO:teuthology.orchestra.run.smithi198.stdout: "is_master": "true", 2021-11-11T21:02:27.739 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:02:27.739 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:02:27.739 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:02:27.740 INFO:teuthology.orchestra.run.smithi198.stdout: "hostnames": [], 2021-11-11T21:02:27.740 INFO:teuthology.orchestra.run.smithi198.stdout: "hostnames_s3website": [], 2021-11-11T21:02:27.740 INFO:teuthology.orchestra.run.smithi198.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:02:27.740 INFO:teuthology.orchestra.run.smithi198.stdout: "zones": [ 2021-11-11T21:02:27.740 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:02:27.741 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:02:27.741 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone2", 2021-11-11T21:02:27.741 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:02:27.741 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi198.front.sepia.ceph.com:8000" 2021-11-11T21:02:27.742 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:02:27.742 INFO:teuthology.orchestra.run.smithi198.stdout: "log_meta": "false", 2021-11-11T21:02:27.742 INFO:teuthology.orchestra.run.smithi198.stdout: "log_data": "true", 2021-11-11T21:02:27.742 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:02:27.742 INFO:teuthology.orchestra.run.smithi198.stdout: "read_only": "false", 2021-11-11T21:02:27.743 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_type": "", 2021-11-11T21:02:27.743 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from_all": "true", 2021-11-11T21:02:27.743 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from": [], 2021-11-11T21:02:27.743 INFO:teuthology.orchestra.run.smithi198.stdout: "redirect_zone": "" 2021-11-11T21:02:27.744 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:02:27.744 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:02:27.744 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:02:27.744 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone1", 2021-11-11T21:02:27.744 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:02:27.745 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:02:27.745 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:02:27.745 INFO:teuthology.orchestra.run.smithi198.stdout: "log_meta": "false", 2021-11-11T21:02:27.745 INFO:teuthology.orchestra.run.smithi198.stdout: "log_data": "true", 2021-11-11T21:02:27.745 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:02:27.746 INFO:teuthology.orchestra.run.smithi198.stdout: "read_only": "false", 2021-11-11T21:02:27.746 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_type": "", 2021-11-11T21:02:27.746 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from_all": "true", 2021-11-11T21:02:27.746 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from": [], 2021-11-11T21:02:27.746 INFO:teuthology.orchestra.run.smithi198.stdout: "redirect_zone": "" 2021-11-11T21:02:27.747 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:27.747 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:02:27.747 INFO:teuthology.orchestra.run.smithi198.stdout: "placement_targets": [ 2021-11-11T21:02:27.747 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:02:27.747 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "default-placement", 2021-11-11T21:02:27.748 INFO:teuthology.orchestra.run.smithi198.stdout: "tags": [], 2021-11-11T21:02:27.748 INFO:teuthology.orchestra.run.smithi198.stdout: "storage_classes": [ 2021-11-11T21:02:27.748 INFO:teuthology.orchestra.run.smithi198.stdout: "STANDARD" 2021-11-11T21:02:27.748 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:02:27.748 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:27.749 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:02:27.749 INFO:teuthology.orchestra.run.smithi198.stdout: "default_placement": "default-placement", 2021-11-11T21:02:27.749 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:02:27.749 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_policy": { 2021-11-11T21:02:27.750 INFO:teuthology.orchestra.run.smithi198.stdout: "groups": [] 2021-11-11T21:02:27.750 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:27.750 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:27.750 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:02:27.750 INFO:teuthology.orchestra.run.smithi198.stdout: "short_zone_ids": [ 2021-11-11T21:02:27.751 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:02:27.751 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:02:27.751 INFO:teuthology.orchestra.run.smithi198.stdout: "val": 1380659451 2021-11-11T21:02:27.751 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:02:27.752 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:02:27.752 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:02:27.752 INFO:teuthology.orchestra.run.smithi198.stdout: "val": 1844113099 2021-11-11T21:02:27.752 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:27.752 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:02:27.753 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:02:27.753 INFO:teuthology.orchestra.run.smithi198.stdout: "master_zonegroup": "test-zonegroup", 2021-11-11T21:02:27.753 INFO:teuthology.orchestra.run.smithi198.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:02:27.753 INFO:teuthology.orchestra.run.smithi198.stdout: "period_config": { 2021-11-11T21:02:27.754 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_quota": { 2021-11-11T21:02:27.754 INFO:teuthology.orchestra.run.smithi198.stdout: "enabled": false, 2021-11-11T21:02:27.754 INFO:teuthology.orchestra.run.smithi198.stdout: "check_on_raw": false, 2021-11-11T21:02:27.754 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size": -1, 2021-11-11T21:02:27.755 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size_kb": 0, 2021-11-11T21:02:27.755 INFO:teuthology.orchestra.run.smithi198.stdout: "max_objects": -1 2021-11-11T21:02:27.755 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:02:27.755 INFO:teuthology.orchestra.run.smithi198.stdout: "user_quota": { 2021-11-11T21:02:27.755 INFO:teuthology.orchestra.run.smithi198.stdout: "enabled": false, 2021-11-11T21:02:27.756 INFO:teuthology.orchestra.run.smithi198.stdout: "check_on_raw": false, 2021-11-11T21:02:27.756 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size": -1, 2021-11-11T21:02:27.756 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size_kb": 0, 2021-11-11T21:02:27.756 INFO:teuthology.orchestra.run.smithi198.stdout: "max_objects": -1 2021-11-11T21:02:27.756 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:02:27.757 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:02:27.757 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:02:27.757 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_name": "test-realm", 2021-11-11T21:02:27.757 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:02:27.757 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:02:27.759 DEBUG:tasks.rgw.c2.client.0:waiting for process to exit 2021-11-11T21:02:27.760 INFO:teuthology.orchestra.run:waiting for 300 2021-11-11T21:02:27.800 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:02:27.801+0000 7f3d4d108700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground (PID: 37622) UID: 0 2021-11-11T21:02:27.801 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:02:27.801+0000 7f3d5e054440 -1 rgw dbstore: shutting down 2021-11-11T21:02:30.679 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 is failed for ~0s 2021-11-11T21:02:33.862 INFO:tasks.rgw.c2.client.0:Stopped 2021-11-11T21:02:33.863 INFO:tasks.rgw.c2.client.0:Restarting daemon 2021-11-11T21:02:33.863 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8000' -n client.0 --cluster c2 -k /etc/ceph/c2.client.0.keyring --log-file /var/log/ceph/rgw.c2.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.0.sock --foreground --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c2.client.0.stdout 2>&1 2021-11-11T21:02:33.866 INFO:tasks.rgw.c2.client.0:Started 2021-11-11T21:02:33.867 DEBUG:teuthology.orchestra.run.smithi198:> curl http://smithi198.front.sepia.ceph.com:8000 2021-11-11T21:02:33.887 INFO:teuthology.orchestra.run.smithi198.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T21:02:33.887 INFO:teuthology.orchestra.run.smithi198.stderr: Dload Upload Total Spent Left Speed 2021-11-11T21:02:36.385 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.0 has been restored 2021-11-11T21:04:44.224 INFO:teuthology.orchestra.run.smithi198.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:30 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:32 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:34 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:36 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:38 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:40 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:42 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:44 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:46 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:48 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:50 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:52 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:54 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:56 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:58 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:00 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:30 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:32 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:34 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:36 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:38 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:40 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:42 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:44 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:46 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:48 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:50 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:52 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:54 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:56 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:58 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:00 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:08 --:--:-- 0curl: (7) Failed to connect to smithi198.front.sepia.ceph.com port 8000: Connection timed out 2021-11-11T21:04:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 7 2021-11-11T21:04:45.228 DEBUG:teuthology.orchestra.run.smithi198:> curl http://smithi198.front.sepia.ceph.com:8000 2021-11-11T21:04:45.251 INFO:teuthology.orchestra.run.smithi198.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T21:04:45.253 INFO:teuthology.orchestra.run.smithi198.stderr: Dload Upload Total Spent Left Speed 2021-11-11T21:04:45.254 INFO:teuthology.orchestra.run.smithi198.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 208k 0 --:--:-- --:--:-- --:--:-- 208k 2021-11-11T21:04:45.255 INFO:teuthology.orchestra.run.smithi198.stdout:anonymous 2021-11-11T21:04:45.255 INFO:tasks.rgw_multisite:creating zone.. 2021-11-11T21:04:45.256 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'create', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--endpoints', 'http://smithi096.front.sepia.ceph.com:8001', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:04:45.257 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'create', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--endpoints', 'http://smithi096.front.sepia.ceph.com:8001', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:04:45.257 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone create --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --access-key UBE0V2NN5ZCDGTQQ --secret H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ --endpoints http://smithi096.front.sepia.ceph.com:8001 --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:04:45.309 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:04:45.310 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:04:45.341 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:04:45.342+0000 7f6002ab4380 0 failed reading obj info from .rgw.root:zone_info.82b05a71-8273-4ec7-b7c6-953c0f40d3e0: (2) No such file or directory 2021-11-11T21:04:45.342 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:04:45.342+0000 7f6002ab4380 0 WARNING: could not read zone params for zone id=82b05a71-8273-4ec7-b7c6-953c0f40d3e0 name=test-zone2 2021-11-11T21:04:45.345 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:04:45.346 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:04:45.346 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone3", 2021-11-11T21:04:45.347 INFO:teuthology.orchestra.run.smithi096.stdout: "domain_root": "test-zone3.rgw.meta:root", 2021-11-11T21:04:45.347 INFO:teuthology.orchestra.run.smithi096.stdout: "control_pool": "test-zone3.rgw.control", 2021-11-11T21:04:45.347 INFO:teuthology.orchestra.run.smithi096.stdout: "gc_pool": "test-zone3.rgw.log:gc", 2021-11-11T21:04:45.347 INFO:teuthology.orchestra.run.smithi096.stdout: "lc_pool": "test-zone3.rgw.log:lc", 2021-11-11T21:04:45.348 INFO:teuthology.orchestra.run.smithi096.stdout: "log_pool": "test-zone3.rgw.log", 2021-11-11T21:04:45.348 INFO:teuthology.orchestra.run.smithi096.stdout: "intent_log_pool": "test-zone3.rgw.log:intent", 2021-11-11T21:04:45.348 INFO:teuthology.orchestra.run.smithi096.stdout: "usage_log_pool": "test-zone3.rgw.log:usage", 2021-11-11T21:04:45.348 INFO:teuthology.orchestra.run.smithi096.stdout: "roles_pool": "test-zone3.rgw.meta:roles", 2021-11-11T21:04:45.349 INFO:teuthology.orchestra.run.smithi096.stdout: "reshard_pool": "test-zone3.rgw.log:reshard", 2021-11-11T21:04:45.349 INFO:teuthology.orchestra.run.smithi096.stdout: "user_keys_pool": "test-zone3.rgw.meta:users.keys", 2021-11-11T21:04:45.349 INFO:teuthology.orchestra.run.smithi096.stdout: "user_email_pool": "test-zone3.rgw.meta:users.email", 2021-11-11T21:04:45.349 INFO:teuthology.orchestra.run.smithi096.stdout: "user_swift_pool": "test-zone3.rgw.meta:users.swift", 2021-11-11T21:04:45.350 INFO:teuthology.orchestra.run.smithi096.stdout: "user_uid_pool": "test-zone3.rgw.meta:users.uid", 2021-11-11T21:04:45.350 INFO:teuthology.orchestra.run.smithi096.stdout: "otp_pool": "test-zone3.rgw.otp", 2021-11-11T21:04:45.350 INFO:teuthology.orchestra.run.smithi096.stdout: "system_key": { 2021-11-11T21:04:45.350 INFO:teuthology.orchestra.run.smithi096.stdout: "access_key": "UBE0V2NN5ZCDGTQQ", 2021-11-11T21:04:45.351 INFO:teuthology.orchestra.run.smithi096.stdout: "secret_key": "H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ" 2021-11-11T21:04:45.351 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:04:45.351 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_pools": [ 2021-11-11T21:04:45.351 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:04:45.352 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "default-placement", 2021-11-11T21:04:45.352 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:04:45.352 INFO:teuthology.orchestra.run.smithi096.stdout: "index_pool": "test-zone3.rgw.buckets.index", 2021-11-11T21:04:45.352 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": { 2021-11-11T21:04:45.352 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD": { 2021-11-11T21:04:45.353 INFO:teuthology.orchestra.run.smithi096.stdout: "data_pool": "test-zone3.rgw.buckets.data" 2021-11-11T21:04:45.353 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:45.353 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:04:45.353 INFO:teuthology.orchestra.run.smithi096.stdout: "data_extra_pool": "test-zone3.rgw.buckets.non-ec", 2021-11-11T21:04:45.354 INFO:teuthology.orchestra.run.smithi096.stdout: "index_type": 0 2021-11-11T21:04:45.354 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:45.354 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:45.354 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:04:45.355 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:04:45.355 INFO:teuthology.orchestra.run.smithi096.stdout: "notif_pool": "test-zone3.rgw.log:notif" 2021-11-11T21:04:45.355 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:04:45.357 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool create test-zone3.rgw.buckets.data 64 64 --cluster c1 2021-11-11T21:04:45.775 INFO:teuthology.orchestra.run.smithi096.stderr:pool 'test-zone3.rgw.buckets.data' created 2021-11-11T21:04:45.791 DEBUG:teuthology.orchestra.run.smithi096:> sudo ceph osd pool application enable test-zone3.rgw.buckets.data rgw --cluster c1 2021-11-11T21:04:46.805 INFO:teuthology.orchestra.run.smithi096.stderr:enabled application 'rgw' on pool 'test-zone3.rgw.buckets.data' 2021-11-11T21:04:46.823 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'placement', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:04:46.824 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'placement', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:04:46.824 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone placement --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:04:46.874 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:04:46.874 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:04:46.902 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:04:46.903 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:04:46.903 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone3", 2021-11-11T21:04:46.903 INFO:teuthology.orchestra.run.smithi096.stdout: "domain_root": "test-zone3.rgw.meta:root", 2021-11-11T21:04:46.904 INFO:teuthology.orchestra.run.smithi096.stdout: "control_pool": "test-zone3.rgw.control", 2021-11-11T21:04:46.904 INFO:teuthology.orchestra.run.smithi096.stdout: "gc_pool": "test-zone3.rgw.log:gc", 2021-11-11T21:04:46.904 INFO:teuthology.orchestra.run.smithi096.stdout: "lc_pool": "test-zone3.rgw.log:lc", 2021-11-11T21:04:46.904 INFO:teuthology.orchestra.run.smithi096.stdout: "log_pool": "test-zone3.rgw.log", 2021-11-11T21:04:46.905 INFO:teuthology.orchestra.run.smithi096.stdout: "intent_log_pool": "test-zone3.rgw.log:intent", 2021-11-11T21:04:46.905 INFO:teuthology.orchestra.run.smithi096.stdout: "usage_log_pool": "test-zone3.rgw.log:usage", 2021-11-11T21:04:46.905 INFO:teuthology.orchestra.run.smithi096.stdout: "roles_pool": "test-zone3.rgw.meta:roles", 2021-11-11T21:04:46.905 INFO:teuthology.orchestra.run.smithi096.stdout: "reshard_pool": "test-zone3.rgw.log:reshard", 2021-11-11T21:04:46.906 INFO:teuthology.orchestra.run.smithi096.stdout: "user_keys_pool": "test-zone3.rgw.meta:users.keys", 2021-11-11T21:04:46.906 INFO:teuthology.orchestra.run.smithi096.stdout: "user_email_pool": "test-zone3.rgw.meta:users.email", 2021-11-11T21:04:46.906 INFO:teuthology.orchestra.run.smithi096.stdout: "user_swift_pool": "test-zone3.rgw.meta:users.swift", 2021-11-11T21:04:46.906 INFO:teuthology.orchestra.run.smithi096.stdout: "user_uid_pool": "test-zone3.rgw.meta:users.uid", 2021-11-11T21:04:46.906 INFO:teuthology.orchestra.run.smithi096.stdout: "otp_pool": "test-zone3.rgw.otp", 2021-11-11T21:04:46.907 INFO:teuthology.orchestra.run.smithi096.stdout: "system_key": { 2021-11-11T21:04:46.907 INFO:teuthology.orchestra.run.smithi096.stdout: "access_key": "UBE0V2NN5ZCDGTQQ", 2021-11-11T21:04:46.907 INFO:teuthology.orchestra.run.smithi096.stdout: "secret_key": "H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ" 2021-11-11T21:04:46.907 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:04:46.908 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_pools": [ 2021-11-11T21:04:46.908 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:04:46.908 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "default-placement", 2021-11-11T21:04:46.908 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:04:46.908 INFO:teuthology.orchestra.run.smithi096.stdout: "index_pool": "test-zone3.rgw.buckets.index", 2021-11-11T21:04:46.909 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": { 2021-11-11T21:04:46.909 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD": { 2021-11-11T21:04:46.909 INFO:teuthology.orchestra.run.smithi096.stdout: "data_pool": "test-zone3.rgw.buckets.data", 2021-11-11T21:04:46.909 INFO:teuthology.orchestra.run.smithi096.stdout: "compression_type": "random" 2021-11-11T21:04:46.909 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:46.910 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:04:46.910 INFO:teuthology.orchestra.run.smithi096.stdout: "data_extra_pool": "test-zone3.rgw.buckets.non-ec", 2021-11-11T21:04:46.910 INFO:teuthology.orchestra.run.smithi096.stdout: "index_type": 0 2021-11-11T21:04:46.910 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:46.911 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:46.911 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:04:46.911 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "", 2021-11-11T21:04:46.911 INFO:teuthology.orchestra.run.smithi096.stdout: "notif_pool": "test-zone3.rgw.log:notif" 2021-11-11T21:04:46.911 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:04:46.913 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:04:46.913 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:04:46.915 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:04:46.955 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:04:46.956 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:04:46.968 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:04:46.968+0000 7ff0bf5c0380 0 period (2aa9d15b-21de-40bc-a7d3-3d5e5933111b does not have zone 4a40cc50-357f-4f11-a692-5925070095e8 configured 2021-11-11T21:04:54.862 INFO:teuthology.orchestra.run.smithi096.stderr:Sending period to new master zone badb2b8d-7821-4be0-b9ef-60c04d880ece 2021-11-11T21:04:54.899 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:04:54.899 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:04:54.900 INFO:teuthology.orchestra.run.smithi096.stdout: "epoch": 3, 2021-11-11T21:04:54.900 INFO:teuthology.orchestra.run.smithi096.stdout: "predecessor_uuid": "ab45263b-38f8-4e1a-9f37-1e83128a94ad", 2021-11-11T21:04:54.900 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": [], 2021-11-11T21:04:54.900 INFO:teuthology.orchestra.run.smithi096.stdout: "period_map": { 2021-11-11T21:04:54.901 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:04:54.901 INFO:teuthology.orchestra.run.smithi096.stdout: "zonegroups": [ 2021-11-11T21:04:54.901 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:04:54.901 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "test-zonegroup", 2021-11-11T21:04:54.902 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zonegroup", 2021-11-11T21:04:54.902 INFO:teuthology.orchestra.run.smithi096.stdout: "api_name": "", 2021-11-11T21:04:54.902 INFO:teuthology.orchestra.run.smithi096.stdout: "is_master": "true", 2021-11-11T21:04:54.902 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:04:54.903 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:04:54.903 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:04:54.903 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames": [], 2021-11-11T21:04:54.904 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames_s3website": [], 2021-11-11T21:04:54.904 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:04:54.904 INFO:teuthology.orchestra.run.smithi096.stdout: "zones": [ 2021-11-11T21:04:54.905 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:04:54.905 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:04:54.906 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone3", 2021-11-11T21:04:54.906 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:04:54.906 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8001" 2021-11-11T21:04:54.907 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:04:54.907 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:04:54.907 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "true", 2021-11-11T21:04:54.908 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:04:54.908 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:04:54.908 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "", 2021-11-11T21:04:54.909 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "true", 2021-11-11T21:04:54.909 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [], 2021-11-11T21:04:54.909 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:04:54.910 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:04:54.910 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:04:54.910 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:04:54.911 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone2", 2021-11-11T21:04:54.911 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:04:54.911 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi198.front.sepia.ceph.com:8000" 2021-11-11T21:04:54.912 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:04:54.912 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:04:54.912 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "true", 2021-11-11T21:04:54.913 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:04:54.913 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:04:54.913 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "", 2021-11-11T21:04:54.914 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "true", 2021-11-11T21:04:54.914 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [], 2021-11-11T21:04:54.914 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:04:54.915 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:04:54.915 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:04:54.915 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:04:54.915 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone1", 2021-11-11T21:04:54.915 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:04:54.916 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:04:54.916 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:04:54.916 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:04:54.917 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "true", 2021-11-11T21:04:54.917 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:04:54.917 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:04:54.917 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "", 2021-11-11T21:04:54.917 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "true", 2021-11-11T21:04:54.918 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [], 2021-11-11T21:04:54.918 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:04:54.918 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:54.919 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:04:54.919 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_targets": [ 2021-11-11T21:04:54.919 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:04:54.920 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "default-placement", 2021-11-11T21:04:54.920 INFO:teuthology.orchestra.run.smithi096.stdout: "tags": [], 2021-11-11T21:04:54.921 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": [ 2021-11-11T21:04:54.921 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD" 2021-11-11T21:04:54.922 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:04:54.922 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:54.923 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:04:54.923 INFO:teuthology.orchestra.run.smithi096.stdout: "default_placement": "default-placement", 2021-11-11T21:04:54.925 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:04:54.925 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_policy": { 2021-11-11T21:04:54.926 INFO:teuthology.orchestra.run.smithi096.stdout: "groups": [] 2021-11-11T21:04:54.926 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:54.926 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:54.927 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:04:54.927 INFO:teuthology.orchestra.run.smithi096.stdout: "short_zone_ids": [ 2021-11-11T21:04:54.927 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:04:54.928 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:04:54.928 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 986920242 2021-11-11T21:04:54.928 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:04:54.928 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:04:54.929 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:04:54.929 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 1380659451 2021-11-11T21:04:54.929 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:04:54.929 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:04:54.930 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:04:54.930 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 1844113099 2021-11-11T21:04:54.930 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:54.930 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:04:54.931 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:04:54.931 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zonegroup": "test-zonegroup", 2021-11-11T21:04:54.931 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:04:54.931 INFO:teuthology.orchestra.run.smithi096.stdout: "period_config": { 2021-11-11T21:04:54.932 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_quota": { 2021-11-11T21:04:54.932 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T21:04:54.932 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T21:04:54.933 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T21:04:54.933 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T21:04:54.933 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T21:04:54.934 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:04:54.934 INFO:teuthology.orchestra.run.smithi096.stdout: "user_quota": { 2021-11-11T21:04:54.934 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T21:04:54.934 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T21:04:54.936 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T21:04:54.936 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T21:04:54.937 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T21:04:54.937 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:04:54.937 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:04:54.938 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:04:54.938 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_name": "test-realm", 2021-11-11T21:04:54.938 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:04:54.939 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:04:54.940 DEBUG:tasks.rgw.c1.client.1:waiting for process to exit 2021-11-11T21:04:54.941 INFO:teuthology.orchestra.run:waiting for 300 2021-11-11T21:04:54.977 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:04:54.978+0000 7f2fa6b48700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8001 -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground (PID: 37283) UID: 0 2021-11-11T21:04:54.978 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:04:54.978+0000 7f2fb7a94440 -1 rgw dbstore: shutting down 2021-11-11T21:05:00.578 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~0s 2021-11-11T21:05:01.044 INFO:tasks.rgw.c1.client.1:Stopped 2021-11-11T21:05:01.044 INFO:tasks.rgw.c1.client.1:Restarting daemon 2021-11-11T21:05:01.045 DEBUG:teuthology.orchestra.run.smithi096:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c1 -k /etc/ceph/c1.client.1.keyring --log-file /var/log/ceph/rgw.c1.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.1.sock --foreground --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c1.client.1.stdout 2>&1 2021-11-11T21:05:01.048 INFO:tasks.rgw.c1.client.1:Started 2021-11-11T21:05:01.048 DEBUG:teuthology.orchestra.run.smithi096:> curl http://smithi096.front.sepia.ceph.com:8001 2021-11-11T21:05:01.068 INFO:teuthology.orchestra.run.smithi096.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T21:05:01.069 INFO:teuthology.orchestra.run.smithi096.stderr: Dload Upload Total Spent Left Speed 2021-11-11T21:05:06.285 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 has been restored 2021-11-11T21:07:10.179 INFO:teuthology.orchestra.run.smithi096.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:30 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:32 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:34 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:36 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:38 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:40 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:42 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:44 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:46 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:48 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:50 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:52 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:54 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:56 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:58 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:00 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:30 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:32 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:34 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:36 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:38 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:40 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:42 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:44 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:46 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:48 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:50 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:52 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:54 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:56 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:58 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:00 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:08 --:--:-- 0curl: (7) Failed to connect to smithi096.front.sepia.ceph.com port 8001: Connection timed out 2021-11-11T21:07:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 7 2021-11-11T21:07:11.183 DEBUG:teuthology.orchestra.run.smithi096:> curl http://smithi096.front.sepia.ceph.com:8001 2021-11-11T21:07:11.205 INFO:teuthology.orchestra.run.smithi096.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T21:07:11.206 INFO:teuthology.orchestra.run.smithi096.stderr: Dload Upload Total Spent Left Speed 2021-11-11T21:07:11.207 INFO:teuthology.orchestra.run.smithi096.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 214 0 214 0 0 208k 0 --:--:-- --:--:-- --:--:-- 208k 2021-11-11T21:07:11.208 INFO:teuthology.orchestra.run.smithi096.stdout:anonymous 2021-11-11T21:07:11.209 INFO:tasks.rgw_multisite:pulling realm configuration to c2 2021-11-11T21:07:11.209 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['realm', 'pull', '--rgw-realm', 'test-realm', '--url', 'http://smithi096.front.sepia.ceph.com:8000', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--url', 'http://smithi096.front.sepia.ceph.com:8000', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:07:11.210 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'realm', 'pull', '--rgw-realm', 'test-realm', '--url', 'http://smithi096.front.sepia.ceph.com:8000', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--url', 'http://smithi096.front.sepia.ceph.com:8000', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:07:11.210 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 realm pull --rgw-realm test-realm --url http://smithi096.front.sepia.ceph.com:8000 --access-key UBE0V2NN5ZCDGTQQ --secret H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ --url http://smithi096.front.sepia.ceph.com:8000 --access-key UBE0V2NN5ZCDGTQQ --secret H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ --cluster c2 --debug-rgw 0 --debug-ms 0 2021-11-11T21:07:11.268 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:07:11.269 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:07:11.308 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:07:11.309+0000 7fc9a7bf2700 -1 rgw realm watcher: RGWRealmWatcher::handle_error oid=realms.31efe32c-6651-482e-86af-2519092e8c22.control err=-107 2021-11-11T21:07:11.326 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:07:11.327 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:07:11.327 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-realm", 2021-11-11T21:07:11.327 INFO:teuthology.orchestra.run.smithi198.stdout: "current_period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:07:11.327 INFO:teuthology.orchestra.run.smithi198.stdout: "epoch": 2 2021-11-11T21:07:11.328 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:07:11.329 INFO:tasks.rgw_multisite:creating zone.. 2021-11-11T21:07:11.330 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'create', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--endpoints', 'http://smithi198.front.sepia.ceph.com:8001', '--tier-type', 'pubsub', '--sync-from-all=0', '--sync-from', 'test-zone1', '--tier-config', 'start_with_full_sync=false,event_retention_days=7', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:07:11.331 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'create', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--access-key', 'UBE0V2NN5ZCDGTQQ', '--secret', 'H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ', '--endpoints', 'http://smithi198.front.sepia.ceph.com:8001', '--tier-type', 'pubsub', '--sync-from-all=0', '--sync-from', 'test-zone1', '--tier-config', 'start_with_full_sync=false,event_retention_days=7', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:07:11.331 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone create --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --access-key UBE0V2NN5ZCDGTQQ --secret H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ --endpoints http://smithi198.front.sepia.ceph.com:8001 --tier-type pubsub --sync-from-all=0 --sync-from test-zone1 --tier-config start_with_full_sync=false,event_retention_days=7 --cluster c2 --debug-rgw 0 --debug-ms 0 2021-11-11T21:07:11.385 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:07:11.385 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:07:11.414 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:07:11.415+0000 7f19be70d380 0 failed reading obj info from .rgw.root:zone_info.4a40cc50-357f-4f11-a692-5925070095e8: (2) No such file or directory 2021-11-11T21:07:11.414 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:07:11.415+0000 7f19be70d380 0 WARNING: could not read zone params for zone id=4a40cc50-357f-4f11-a692-5925070095e8 name=test-zone3 2021-11-11T21:07:11.415 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:07:11.415+0000 7f19be70d380 0 failed reading obj info from .rgw.root:zone_info.badb2b8d-7821-4be0-b9ef-60c04d880ece: (2) No such file or directory 2021-11-11T21:07:11.415 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:07:11.415+0000 7f19be70d380 0 WARNING: could not read zone params for zone id=badb2b8d-7821-4be0-b9ef-60c04d880ece name=test-zone1 2021-11-11T21:07:11.419 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:07:11.419 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "31f4b03b-e9bc-4500-a988-e29031fe0663", 2021-11-11T21:07:11.419 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone4", 2021-11-11T21:07:11.419 INFO:teuthology.orchestra.run.smithi198.stdout: "domain_root": "test-zone4.rgw.meta:root", 2021-11-11T21:07:11.420 INFO:teuthology.orchestra.run.smithi198.stdout: "control_pool": "test-zone4.rgw.control", 2021-11-11T21:07:11.420 INFO:teuthology.orchestra.run.smithi198.stdout: "gc_pool": "test-zone4.rgw.log:gc", 2021-11-11T21:07:11.420 INFO:teuthology.orchestra.run.smithi198.stdout: "lc_pool": "test-zone4.rgw.log:lc", 2021-11-11T21:07:11.420 INFO:teuthology.orchestra.run.smithi198.stdout: "log_pool": "test-zone4.rgw.log", 2021-11-11T21:07:11.420 INFO:teuthology.orchestra.run.smithi198.stdout: "intent_log_pool": "test-zone4.rgw.log:intent", 2021-11-11T21:07:11.421 INFO:teuthology.orchestra.run.smithi198.stdout: "usage_log_pool": "test-zone4.rgw.log:usage", 2021-11-11T21:07:11.421 INFO:teuthology.orchestra.run.smithi198.stdout: "roles_pool": "test-zone4.rgw.meta:roles", 2021-11-11T21:07:11.421 INFO:teuthology.orchestra.run.smithi198.stdout: "reshard_pool": "test-zone4.rgw.log:reshard", 2021-11-11T21:07:11.421 INFO:teuthology.orchestra.run.smithi198.stdout: "user_keys_pool": "test-zone4.rgw.meta:users.keys", 2021-11-11T21:07:11.422 INFO:teuthology.orchestra.run.smithi198.stdout: "user_email_pool": "test-zone4.rgw.meta:users.email", 2021-11-11T21:07:11.422 INFO:teuthology.orchestra.run.smithi198.stdout: "user_swift_pool": "test-zone4.rgw.meta:users.swift", 2021-11-11T21:07:11.422 INFO:teuthology.orchestra.run.smithi198.stdout: "user_uid_pool": "test-zone4.rgw.meta:users.uid", 2021-11-11T21:07:11.422 INFO:teuthology.orchestra.run.smithi198.stdout: "otp_pool": "test-zone4.rgw.otp", 2021-11-11T21:07:11.422 INFO:teuthology.orchestra.run.smithi198.stdout: "system_key": { 2021-11-11T21:07:11.423 INFO:teuthology.orchestra.run.smithi198.stdout: "access_key": "UBE0V2NN5ZCDGTQQ", 2021-11-11T21:07:11.423 INFO:teuthology.orchestra.run.smithi198.stdout: "secret_key": "H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ" 2021-11-11T21:07:11.423 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:11.423 INFO:teuthology.orchestra.run.smithi198.stdout: "placement_pools": [ 2021-11-11T21:07:11.423 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:11.424 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "default-placement", 2021-11-11T21:07:11.424 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:07:11.424 INFO:teuthology.orchestra.run.smithi198.stdout: "index_pool": "test-zone4.rgw.buckets.index", 2021-11-11T21:07:11.424 INFO:teuthology.orchestra.run.smithi198.stdout: "storage_classes": { 2021-11-11T21:07:11.424 INFO:teuthology.orchestra.run.smithi198.stdout: "STANDARD": { 2021-11-11T21:07:11.425 INFO:teuthology.orchestra.run.smithi198.stdout: "data_pool": "test-zone4.rgw.buckets.data" 2021-11-11T21:07:11.425 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:11.425 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:11.425 INFO:teuthology.orchestra.run.smithi198.stdout: "data_extra_pool": "test-zone4.rgw.buckets.non-ec", 2021-11-11T21:07:11.425 INFO:teuthology.orchestra.run.smithi198.stdout: "index_type": 0 2021-11-11T21:07:11.426 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:11.426 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:11.426 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:07:11.426 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_config": { 2021-11-11T21:07:11.427 INFO:teuthology.orchestra.run.smithi198.stdout: "event_retention_days": 7, 2021-11-11T21:07:11.427 INFO:teuthology.orchestra.run.smithi198.stdout: "start_with_full_sync": false 2021-11-11T21:07:11.427 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:11.427 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:07:11.427 INFO:teuthology.orchestra.run.smithi198.stdout: "notif_pool": "test-zone4.rgw.log:notif" 2021-11-11T21:07:11.428 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:07:11.429 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool create test-zone4.rgw.buckets.data 64 64 --cluster c2 2021-11-11T21:07:11.788 INFO:teuthology.orchestra.run.smithi198.stderr:pool 'test-zone4.rgw.buckets.data' created 2021-11-11T21:07:11.805 DEBUG:teuthology.orchestra.run.smithi198:> sudo ceph osd pool application enable test-zone4.rgw.buckets.data rgw --cluster c2 2021-11-11T21:07:12.819 INFO:teuthology.orchestra.run.smithi198.stderr:enabled application 'rgw' on pool 'test-zone4.rgw.buckets.data' 2021-11-11T21:07:12.834 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'placement', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:07:12.835 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'placement', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', 'modify', '--placement-id', 'default-placement', '--compression', 'random', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:07:12.835 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone placement --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm modify --placement-id default-placement --compression random --cluster c2 --debug-rgw 0 --debug-ms 0 2021-11-11T21:07:12.883 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:07:12.884 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:07:12.912 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:07:12.912 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "31f4b03b-e9bc-4500-a988-e29031fe0663", 2021-11-11T21:07:12.913 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone4", 2021-11-11T21:07:12.913 INFO:teuthology.orchestra.run.smithi198.stdout: "domain_root": "test-zone4.rgw.meta:root", 2021-11-11T21:07:12.913 INFO:teuthology.orchestra.run.smithi198.stdout: "control_pool": "test-zone4.rgw.control", 2021-11-11T21:07:12.913 INFO:teuthology.orchestra.run.smithi198.stdout: "gc_pool": "test-zone4.rgw.log:gc", 2021-11-11T21:07:12.913 INFO:teuthology.orchestra.run.smithi198.stdout: "lc_pool": "test-zone4.rgw.log:lc", 2021-11-11T21:07:12.914 INFO:teuthology.orchestra.run.smithi198.stdout: "log_pool": "test-zone4.rgw.log", 2021-11-11T21:07:12.914 INFO:teuthology.orchestra.run.smithi198.stdout: "intent_log_pool": "test-zone4.rgw.log:intent", 2021-11-11T21:07:12.914 INFO:teuthology.orchestra.run.smithi198.stdout: "usage_log_pool": "test-zone4.rgw.log:usage", 2021-11-11T21:07:12.914 INFO:teuthology.orchestra.run.smithi198.stdout: "roles_pool": "test-zone4.rgw.meta:roles", 2021-11-11T21:07:12.915 INFO:teuthology.orchestra.run.smithi198.stdout: "reshard_pool": "test-zone4.rgw.log:reshard", 2021-11-11T21:07:12.915 INFO:teuthology.orchestra.run.smithi198.stdout: "user_keys_pool": "test-zone4.rgw.meta:users.keys", 2021-11-11T21:07:12.915 INFO:teuthology.orchestra.run.smithi198.stdout: "user_email_pool": "test-zone4.rgw.meta:users.email", 2021-11-11T21:07:12.915 INFO:teuthology.orchestra.run.smithi198.stdout: "user_swift_pool": "test-zone4.rgw.meta:users.swift", 2021-11-11T21:07:12.915 INFO:teuthology.orchestra.run.smithi198.stdout: "user_uid_pool": "test-zone4.rgw.meta:users.uid", 2021-11-11T21:07:12.916 INFO:teuthology.orchestra.run.smithi198.stdout: "otp_pool": "test-zone4.rgw.otp", 2021-11-11T21:07:12.916 INFO:teuthology.orchestra.run.smithi198.stdout: "system_key": { 2021-11-11T21:07:12.916 INFO:teuthology.orchestra.run.smithi198.stdout: "access_key": "UBE0V2NN5ZCDGTQQ", 2021-11-11T21:07:12.916 INFO:teuthology.orchestra.run.smithi198.stdout: "secret_key": "H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ" 2021-11-11T21:07:12.917 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:12.917 INFO:teuthology.orchestra.run.smithi198.stdout: "placement_pools": [ 2021-11-11T21:07:12.917 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:12.917 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "default-placement", 2021-11-11T21:07:12.918 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:07:12.918 INFO:teuthology.orchestra.run.smithi198.stdout: "index_pool": "test-zone4.rgw.buckets.index", 2021-11-11T21:07:12.918 INFO:teuthology.orchestra.run.smithi198.stdout: "storage_classes": { 2021-11-11T21:07:12.918 INFO:teuthology.orchestra.run.smithi198.stdout: "STANDARD": { 2021-11-11T21:07:12.918 INFO:teuthology.orchestra.run.smithi198.stdout: "data_pool": "test-zone4.rgw.buckets.data", 2021-11-11T21:07:12.919 INFO:teuthology.orchestra.run.smithi198.stdout: "compression_type": "random" 2021-11-11T21:07:12.919 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:12.919 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:12.919 INFO:teuthology.orchestra.run.smithi198.stdout: "data_extra_pool": "test-zone4.rgw.buckets.non-ec", 2021-11-11T21:07:12.920 INFO:teuthology.orchestra.run.smithi198.stdout: "index_type": 0 2021-11-11T21:07:12.920 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:12.920 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:12.920 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:07:12.920 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_config": { 2021-11-11T21:07:12.921 INFO:teuthology.orchestra.run.smithi198.stdout: "event_retention_days": 7, 2021-11-11T21:07:12.921 INFO:teuthology.orchestra.run.smithi198.stdout: "start_with_full_sync": false 2021-11-11T21:07:12.921 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:12.921 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "", 2021-11-11T21:07:12.922 INFO:teuthology.orchestra.run.smithi198.stdout: "notif_pool": "test-zone4.rgw.log:notif" 2021-11-11T21:07:12.922 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:07:12.923 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:07:12.924 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:07:12.924 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2021-11-11T21:07:12.965 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:07:12.965 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:07:12.974 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:07:12.975+0000 7f11f4082380 0 period (2aa9d15b-21de-40bc-a7d3-3d5e5933111b does not have zone 31f4b03b-e9bc-4500-a988-e29031fe0663 configured 2021-11-11T21:07:20.888 INFO:teuthology.orchestra.run.smithi198.stderr:Sending period to new master zone badb2b8d-7821-4be0-b9ef-60c04d880ece 2021-11-11T21:07:20.921 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:07:20.921 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:07:20.921 INFO:teuthology.orchestra.run.smithi198.stdout: "epoch": 4, 2021-11-11T21:07:20.922 INFO:teuthology.orchestra.run.smithi198.stdout: "predecessor_uuid": "ab45263b-38f8-4e1a-9f37-1e83128a94ad", 2021-11-11T21:07:20.922 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": [], 2021-11-11T21:07:20.922 INFO:teuthology.orchestra.run.smithi198.stdout: "period_map": { 2021-11-11T21:07:20.922 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:07:20.923 INFO:teuthology.orchestra.run.smithi198.stdout: "zonegroups": [ 2021-11-11T21:07:20.923 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:20.923 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "test-zonegroup", 2021-11-11T21:07:20.923 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zonegroup", 2021-11-11T21:07:20.923 INFO:teuthology.orchestra.run.smithi198.stdout: "api_name": "", 2021-11-11T21:07:20.924 INFO:teuthology.orchestra.run.smithi198.stdout: "is_master": "true", 2021-11-11T21:07:20.924 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:07:20.924 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:07:20.924 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:07:20.925 INFO:teuthology.orchestra.run.smithi198.stdout: "hostnames": [], 2021-11-11T21:07:20.925 INFO:teuthology.orchestra.run.smithi198.stdout: "hostnames_s3website": [], 2021-11-11T21:07:20.925 INFO:teuthology.orchestra.run.smithi198.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:07:20.925 INFO:teuthology.orchestra.run.smithi198.stdout: "zones": [ 2021-11-11T21:07:20.925 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:20.926 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "31f4b03b-e9bc-4500-a988-e29031fe0663", 2021-11-11T21:07:20.926 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone4", 2021-11-11T21:07:20.926 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:07:20.926 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi198.front.sepia.ceph.com:8001" 2021-11-11T21:07:20.927 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:07:20.927 INFO:teuthology.orchestra.run.smithi198.stdout: "log_meta": "false", 2021-11-11T21:07:20.927 INFO:teuthology.orchestra.run.smithi198.stdout: "log_data": "true", 2021-11-11T21:07:20.927 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:07:20.927 INFO:teuthology.orchestra.run.smithi198.stdout: "read_only": "false", 2021-11-11T21:07:20.928 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_type": "pubsub", 2021-11-11T21:07:20.928 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from_all": "false", 2021-11-11T21:07:20.928 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from": [ 2021-11-11T21:07:20.928 INFO:teuthology.orchestra.run.smithi198.stdout: "test-zone1" 2021-11-11T21:07:20.929 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:07:20.929 INFO:teuthology.orchestra.run.smithi198.stdout: "redirect_zone": "" 2021-11-11T21:07:20.929 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:20.929 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:20.930 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:07:20.930 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone3", 2021-11-11T21:07:20.930 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:07:20.930 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi096.front.sepia.ceph.com:8001" 2021-11-11T21:07:20.930 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:07:20.931 INFO:teuthology.orchestra.run.smithi198.stdout: "log_meta": "false", 2021-11-11T21:07:20.931 INFO:teuthology.orchestra.run.smithi198.stdout: "log_data": "true", 2021-11-11T21:07:20.931 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:07:20.931 INFO:teuthology.orchestra.run.smithi198.stdout: "read_only": "false", 2021-11-11T21:07:20.932 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_type": "", 2021-11-11T21:07:20.932 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from_all": "true", 2021-11-11T21:07:20.932 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from": [], 2021-11-11T21:07:20.932 INFO:teuthology.orchestra.run.smithi198.stdout: "redirect_zone": "" 2021-11-11T21:07:20.933 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:20.933 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:20.933 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:07:20.933 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone2", 2021-11-11T21:07:20.934 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:07:20.934 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi198.front.sepia.ceph.com:8000" 2021-11-11T21:07:20.934 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:07:20.934 INFO:teuthology.orchestra.run.smithi198.stdout: "log_meta": "false", 2021-11-11T21:07:20.934 INFO:teuthology.orchestra.run.smithi198.stdout: "log_data": "true", 2021-11-11T21:07:20.935 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:07:20.935 INFO:teuthology.orchestra.run.smithi198.stdout: "read_only": "false", 2021-11-11T21:07:20.935 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_type": "", 2021-11-11T21:07:20.935 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from_all": "true", 2021-11-11T21:07:20.936 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from": [], 2021-11-11T21:07:20.936 INFO:teuthology.orchestra.run.smithi198.stdout: "redirect_zone": "" 2021-11-11T21:07:20.936 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:20.936 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:20.936 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:07:20.937 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone1", 2021-11-11T21:07:20.937 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:07:20.937 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:07:20.937 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:07:20.938 INFO:teuthology.orchestra.run.smithi198.stdout: "log_meta": "false", 2021-11-11T21:07:20.938 INFO:teuthology.orchestra.run.smithi198.stdout: "log_data": "true", 2021-11-11T21:07:20.938 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:07:20.938 INFO:teuthology.orchestra.run.smithi198.stdout: "read_only": "false", 2021-11-11T21:07:20.938 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_type": "", 2021-11-11T21:07:20.939 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from_all": "true", 2021-11-11T21:07:20.939 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from": [], 2021-11-11T21:07:20.939 INFO:teuthology.orchestra.run.smithi198.stdout: "redirect_zone": "" 2021-11-11T21:07:20.939 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:20.940 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:07:20.940 INFO:teuthology.orchestra.run.smithi198.stdout: "placement_targets": [ 2021-11-11T21:07:20.940 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:20.940 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "default-placement", 2021-11-11T21:07:20.940 INFO:teuthology.orchestra.run.smithi198.stdout: "tags": [], 2021-11-11T21:07:20.941 INFO:teuthology.orchestra.run.smithi198.stdout: "storage_classes": [ 2021-11-11T21:07:20.941 INFO:teuthology.orchestra.run.smithi198.stdout: "STANDARD" 2021-11-11T21:07:20.941 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:07:20.941 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:20.942 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:07:20.942 INFO:teuthology.orchestra.run.smithi198.stdout: "default_placement": "default-placement", 2021-11-11T21:07:20.942 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:07:20.942 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_policy": { 2021-11-11T21:07:20.943 INFO:teuthology.orchestra.run.smithi198.stdout: "groups": [] 2021-11-11T21:07:20.943 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:20.943 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:20.943 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:07:20.943 INFO:teuthology.orchestra.run.smithi198.stdout: "short_zone_ids": [ 2021-11-11T21:07:20.944 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:20.945 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "31f4b03b-e9bc-4500-a988-e29031fe0663", 2021-11-11T21:07:20.945 INFO:teuthology.orchestra.run.smithi198.stdout: "val": 2936728019 2021-11-11T21:07:20.945 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:20.945 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:20.946 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:07:20.946 INFO:teuthology.orchestra.run.smithi198.stdout: "val": 986920242 2021-11-11T21:07:20.946 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:20.946 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:20.947 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:07:20.947 INFO:teuthology.orchestra.run.smithi198.stdout: "val": 1380659451 2021-11-11T21:07:20.947 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:20.947 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:07:20.948 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:07:20.948 INFO:teuthology.orchestra.run.smithi198.stdout: "val": 1844113099 2021-11-11T21:07:20.948 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:20.948 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:07:20.948 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:20.949 INFO:teuthology.orchestra.run.smithi198.stdout: "master_zonegroup": "test-zonegroup", 2021-11-11T21:07:20.949 INFO:teuthology.orchestra.run.smithi198.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:07:20.949 INFO:teuthology.orchestra.run.smithi198.stdout: "period_config": { 2021-11-11T21:07:20.949 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_quota": { 2021-11-11T21:07:20.950 INFO:teuthology.orchestra.run.smithi198.stdout: "enabled": false, 2021-11-11T21:07:20.950 INFO:teuthology.orchestra.run.smithi198.stdout: "check_on_raw": false, 2021-11-11T21:07:20.950 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size": -1, 2021-11-11T21:07:20.950 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size_kb": 0, 2021-11-11T21:07:20.950 INFO:teuthology.orchestra.run.smithi198.stdout: "max_objects": -1 2021-11-11T21:07:20.951 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:20.951 INFO:teuthology.orchestra.run.smithi198.stdout: "user_quota": { 2021-11-11T21:07:20.951 INFO:teuthology.orchestra.run.smithi198.stdout: "enabled": false, 2021-11-11T21:07:20.951 INFO:teuthology.orchestra.run.smithi198.stdout: "check_on_raw": false, 2021-11-11T21:07:20.952 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size": -1, 2021-11-11T21:07:20.952 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size_kb": 0, 2021-11-11T21:07:20.952 INFO:teuthology.orchestra.run.smithi198.stdout: "max_objects": -1 2021-11-11T21:07:20.953 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:07:20.953 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:07:20.953 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:07:20.953 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_name": "test-realm", 2021-11-11T21:07:20.954 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:07:20.954 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:07:20.954 DEBUG:tasks.rgw.c2.client.1:waiting for process to exit 2021-11-11T21:07:20.955 INFO:teuthology.orchestra.run:waiting for 300 2021-11-11T21:07:20.986 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:07:20.987+0000 7f76b2c3f700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8001 -n client.1 --cluster c2 -k /etc/ceph/c2.client.1.keyring --log-file /var/log/ceph/rgw.c2.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.1.sock --foreground (PID: 37625) UID: 0 2021-11-11T21:07:20.987 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:07:20.987+0000 7f76c3b8b440 -1 rgw dbstore: shutting down 2021-11-11T21:07:21.577 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.1 is failed for ~0s 2021-11-11T21:07:27.057 INFO:tasks.rgw.c2.client.1:Stopped 2021-11-11T21:07:27.058 INFO:tasks.rgw.c2.client.1:Restarting daemon 2021-11-11T21:07:27.058 DEBUG:teuthology.orchestra.run.smithi198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term radosgw --rgw-frontends 'beast port=8001' -n client.1 --cluster c2 -k /etc/ceph/c2.client.1.keyring --log-file /var/log/ceph/rgw.c2.client.1.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c2.client.1.sock --foreground --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm | sudo tee /var/log/ceph/rgw.c2.client.1.stdout 2>&1 2021-11-11T21:07:27.061 INFO:tasks.rgw.c2.client.1:Started 2021-11-11T21:07:27.062 DEBUG:teuthology.orchestra.run.smithi198:> curl http://smithi198.front.sepia.ceph.com:8001 2021-11-11T21:07:27.082 INFO:teuthology.orchestra.run.smithi198.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T21:07:27.083 INFO:teuthology.orchestra.run.smithi198.stderr: Dload Upload Total Spent Left Speed 2021-11-11T21:07:27.284 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c2.rgw.client.1 has been restored 2021-11-11T21:09:39.423 INFO:teuthology.orchestra.run.smithi198.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:30 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:32 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:34 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:36 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:38 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:40 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:42 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:44 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:46 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:48 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:50 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:52 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:54 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:56 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:58 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:00 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:30 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:32 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:34 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:36 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:38 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:40 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:42 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:44 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:46 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:48 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:50 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:52 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:54 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:56 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:01:58 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:00 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:02:10 --:--:-- 0curl: (7) Failed to connect to smithi198.front.sepia.ceph.com port 8001: Connection timed out 2021-11-11T21:09:39.425 DEBUG:teuthology.orchestra.run:got remote process result: 7 2021-11-11T21:09:40.427 DEBUG:teuthology.orchestra.run.smithi198:> curl http://smithi198.front.sepia.ceph.com:8001 2021-11-11T21:09:40.452 INFO:teuthology.orchestra.run.smithi198.stderr: % Total % Received % Xferd Average Speed Time Time Time Current 2021-11-11T21:09:40.453 INFO:teuthology.orchestra.run.smithi198.stderr: Dload Upload Total Spent Left Speed 2021-11-11T21:09:40.455 INFO:teuthology.orchestra.run.smithi198.stderr: 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 134 100 134 0 0 130k 0 --:--:-- --:--:-- --:--:-- 130k 2021-11-11T21:09:40.456 INFO:teuthology.orchestra.run.smithi198.stdout:{"Code":"MethodNotAllowed","RequestId":"tx000004f7e7700b174ffc7-00618d8694-10e0-test-zone4","HostId":"10e0-test-zone4-test-zonegroup"} 2021-11-11T21:09:40.457 INFO:tasks.rgw_multisite:rgw multisite configuration completed 2021-11-11T21:09:40.458 INFO:teuthology.run_tasks:Running task rgw-multisite-tests... 2021-11-11T21:09:40.480 INFO:tasks.rgw_multisite_tests:importing tests from /home/teuthworker/src/git.ceph.com_ceph-c_a3a8018ba3cfe536252b397411bf18b7828c6098/qa/../src/test/rgw/rgw_multi 2021-11-11T21:09:40.500 INFO:tasks.rgw_multisite_tests:creating test user.. 2021-11-11T21:09:40.501 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['user', 'create', '--uid', 'rgw-multisite-test-user', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--display-name', 'Multisite Test User', '--gen-access-key', '--gen-secret', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:09:40.501 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'user', 'create', '--uid', 'rgw-multisite-test-user', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--display-name', 'Multisite Test User', '--gen-access-key', '--gen-secret', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:09:40.502 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 user create --uid rgw-multisite-test-user --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --display-name 'Multisite Test User' --gen-access-key --gen-secret --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:09:40.556 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:40.556 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:40.618 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:09:40.619 INFO:teuthology.orchestra.run.smithi096.stdout: "user_id": "rgw-multisite-test-user", 2021-11-11T21:09:40.619 INFO:teuthology.orchestra.run.smithi096.stdout: "display_name": "Multisite Test User", 2021-11-11T21:09:40.620 INFO:teuthology.orchestra.run.smithi096.stdout: "email": "", 2021-11-11T21:09:40.620 INFO:teuthology.orchestra.run.smithi096.stdout: "suspended": 0, 2021-11-11T21:09:40.621 INFO:teuthology.orchestra.run.smithi096.stdout: "max_buckets": 1000, 2021-11-11T21:09:40.621 INFO:teuthology.orchestra.run.smithi096.stdout: "subusers": [], 2021-11-11T21:09:40.621 INFO:teuthology.orchestra.run.smithi096.stdout: "keys": [ 2021-11-11T21:09:40.622 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:40.622 INFO:teuthology.orchestra.run.smithi096.stdout: "user": "rgw-multisite-test-user", 2021-11-11T21:09:40.622 INFO:teuthology.orchestra.run.smithi096.stdout: "access_key": "VZIDLOJZFEVKLEGPK68V", 2021-11-11T21:09:40.623 INFO:teuthology.orchestra.run.smithi096.stdout: "secret_key": "1myR9jCVny1l46n2aMabDzZPxgiIEECD3YYonIwf" 2021-11-11T21:09:40.623 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:40.624 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:09:40.624 INFO:teuthology.orchestra.run.smithi096.stdout: "swift_keys": [], 2021-11-11T21:09:40.625 INFO:teuthology.orchestra.run.smithi096.stdout: "caps": [], 2021-11-11T21:09:40.625 INFO:teuthology.orchestra.run.smithi096.stdout: "op_mask": "read, write, delete", 2021-11-11T21:09:40.625 INFO:teuthology.orchestra.run.smithi096.stdout: "default_placement": "", 2021-11-11T21:09:40.626 INFO:teuthology.orchestra.run.smithi096.stdout: "default_storage_class": "", 2021-11-11T21:09:40.626 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_tags": [], 2021-11-11T21:09:40.626 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_quota": { 2021-11-11T21:09:40.627 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T21:09:40.627 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T21:09:40.627 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T21:09:40.628 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T21:09:40.628 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T21:09:40.629 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:40.629 INFO:teuthology.orchestra.run.smithi096.stdout: "user_quota": { 2021-11-11T21:09:40.629 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T21:09:40.630 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T21:09:40.630 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T21:09:40.631 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T21:09:40.631 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T21:09:40.632 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:40.632 INFO:teuthology.orchestra.run.smithi096.stdout: "temp_url_keys": [], 2021-11-11T21:09:40.632 INFO:teuthology.orchestra.run.smithi096.stdout: "type": "rgw", 2021-11-11T21:09:40.633 INFO:teuthology.orchestra.run.smithi096.stdout: "mfa_ids": [] 2021-11-11T21:09:40.633 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:09:40.633 INFO:teuthology.orchestra.run.smithi096.stdout: 2021-11-11T21:09:40.638 INFO:rgw_multi.tests:meta checkpoint 2021-11-11T21:09:40.639 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:40.639 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:40.640 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:40.681 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:40.682 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:40.731 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:09:40.747 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:09:40.747 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:40.748 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:09:40.748 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:09:40.748 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:40.748 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:40.749 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:40.749 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:09:40.749 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:40.749 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:40.750 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:40.750 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:09:40.750 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:40.751 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:40.751 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:40.751 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:09:40.751 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:40.751 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:09:40.753 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:09:40.753 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:40.754 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:40.754 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:40.803 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:40.804 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:40.884 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:09:40.884 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:09:40.884 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:09:40.885 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:09:40.885 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:09:40.885 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:09:40.885 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:40.886 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:40.886 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:09:40.886 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:40.887 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:09:40.888 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:40.888 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:40.888 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:40.888 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:40.889 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:09:40.889 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:40.889 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:40.889 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:40.889 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:40.890 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:40.890 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:40.890 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:09:40.890 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:40.891 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:40.891 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:40.891 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:40.891 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:40.892 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:40.892 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:40.892 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:40.892 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:40.893 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:40.893 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:40.893 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:09:40.894 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:40.894 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:40.894 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:40.894 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:40.895 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:40.895 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:40.895 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:40.895 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:40.896 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:40.896 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:40.896 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:40.896 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:09:40.896 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:40.897 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:40.898 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:40.898 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:40.898 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:40.899 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:40.899 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:40.899 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:40.899 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:40.900 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:40.900 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:09:40.900 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:40.900 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:09:40.901 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:09:40.901 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:09:40.901 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:40.901 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:09:40.903 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=0 master=1_1636664980.616234_46.1 target= 2021-11-11T21:09:45.904 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:45.905 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:45.905 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:45.954 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:45.955 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:46.031 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:09:46.031 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:09:46.032 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:09:46.032 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:09:46.032 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:09:46.032 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:09:46.033 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:46.033 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:46.033 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:09:46.033 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:46.033 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:09:46.034 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:46.034 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:46.034 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:09:46.034 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:46.035 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:09:46.035 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:46.035 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:09:46.036 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:46.036 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.036 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:46.036 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:46.037 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:09:46.037 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:46.037 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:46.037 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:46.038 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:46.038 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:46.038 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:46.038 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:46.039 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:46.039 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.039 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:46.039 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:46.040 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:09:46.040 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:46.040 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:46.040 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:46.041 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:46.041 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:46.041 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:46.041 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:46.042 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:46.042 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.042 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:46.042 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:46.042 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:09:46.043 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:46.043 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:46.043 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:46.044 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:46.044 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:46.044 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:46.044 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:46.044 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:46.045 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.045 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.045 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:09:46.045 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:46.046 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:09:46.046 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:09:46.046 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:09:46.046 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.047 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:09:46.048 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:09:46.048 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:09:46.049 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:46.049 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:46.049 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:46.098 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:46.098 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:46.164 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:09:46.164 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:09:46.165 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:09:46.165 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:09:46.165 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:09:46.165 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:09:46.165 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:09:46.166 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:46.166 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:09:46.167 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:46.167 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:09:46.167 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:09:46.167 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:09:46.168 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:09:46.168 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:09:46.168 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:09:46.168 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:09:46.169 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:09:46.169 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:09:46.169 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:46.169 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:46.170 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:46.171 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:09:46.171 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:09:46.171 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:09:46.171 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:46.172 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:09:46.172 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:09:46.172 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:09:46.172 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:09:46.173 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:09:46.173 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:46.173 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:46.174 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:46.174 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:09:46.175 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:09:46.175 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:09:46.176 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:46.176 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:09:46.177 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:09:46.178 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:09:46.178 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:09:46.179 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:09:46.179 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:46.179 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:46.180 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:46.180 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:09:46.181 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:09:46.181 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:09:46.181 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:46.182 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:09:46.182 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:09:46.183 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:09:46.183 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:09:46.183 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:09:46.184 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:46.184 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:46.184 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:09:46.184 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:46.185 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:09:46.185 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:09:46.185 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:09:46.185 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:46.186 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:09:46.188 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:09:46.188 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:09:46.189 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:46.190 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:46.191 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:46.235 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:46.236 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:46.295 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:09:46.295 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:09:46.296 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:09:46.296 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:09:46.296 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:09:46.296 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:09:46.296 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:46.297 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:46.297 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:09:46.297 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:46.297 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:09:46.298 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:46.298 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:46.298 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:46.298 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:46.298 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:09:46.299 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:46.299 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:46.299 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:46.299 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.300 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:46.300 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:46.300 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:09:46.300 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:46.300 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:46.301 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:46.301 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:46.301 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:46.301 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:46.302 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:46.302 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:46.302 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.302 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:46.302 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:46.303 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:09:46.303 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:46.303 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:46.303 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:46.304 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:46.304 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:46.304 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:46.304 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:46.304 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:46.305 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.305 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:46.305 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:46.305 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:09:46.306 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:46.306 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:46.306 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:46.306 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:46.306 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:46.307 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:46.307 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:46.307 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:46.307 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.307 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.308 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:09:46.308 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:46.308 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:09:46.308 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:09:46.309 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:09:46.309 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:46.309 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:09:46.310 WARNING:rgw_multi.tests:zone test-zone4 behind master: shard=0 master=1_1636664980.616234_46.1 target= 2021-11-11T21:09:51.312 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:51.312 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:51.313 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:51.361 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:51.361 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:51.430 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:09:51.431 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:09:51.431 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:09:51.431 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:09:51.432 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:09:51.432 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:09:51.432 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.432 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.433 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:09:51.433 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:51.433 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:09:51.433 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:51.433 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:51.434 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:09:51.434 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:51.434 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:09:51.434 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:51.434 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:09:51.435 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.435 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.435 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.435 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:51.436 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:09:51.436 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:51.436 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:51.436 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:51.436 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:51.437 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:51.437 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:51.437 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:51.437 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.437 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.438 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.438 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:51.438 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:09:51.438 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:51.439 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:51.439 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:51.439 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:51.439 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:51.439 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:51.440 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:51.440 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.440 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.440 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.440 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:51.441 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:09:51.441 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:51.441 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:51.441 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:09:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:09:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:09:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:09:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.444 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:09:51.446 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:09:51.446 INFO:rgw_multi.tests:meta checkpoint 2021-11-11T21:09:51.447 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:51.447 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:51.447 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:51.495 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:51.495 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:51.571 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:09:51.596 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:09:51.596 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:51.596 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:09:51.597 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:09:51.599 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:51.601 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:51.601 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:51.601 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:09:51.601 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:51.602 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:51.602 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:51.602 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:09:51.602 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:51.603 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:51.603 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:51.603 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:09:51.603 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:51.604 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:09:51.605 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:09:51.605 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:51.605 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:51.606 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:51.656 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:51.657 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:51.733 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:09:51.734 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:09:51.734 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:09:51.734 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:09:51.734 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:09:51.735 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:09:51.735 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.735 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.735 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:09:51.736 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:51.736 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:09:51.736 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:51.736 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:51.736 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:09:51.737 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:51.737 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:09:51.737 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:51.737 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:09:51.738 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.738 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.738 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.738 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:51.739 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:09:51.739 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:51.739 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:51.739 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:51.740 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:51.740 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:51.740 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:51.740 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:51.740 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.741 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.741 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.741 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:51.741 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:09:51.741 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:51.742 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:51.742 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:51.742 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:51.742 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:51.743 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:51.743 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:51.743 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.743 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.744 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.744 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:51.744 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:09:51.744 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:51.745 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:51.745 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:51.745 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:51.745 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:51.745 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:51.746 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:51.746 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.746 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.746 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.747 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:09:51.747 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.747 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:09:51.747 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:09:51.748 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:09:51.748 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.748 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:09:51.749 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:09:51.750 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:09:51.750 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:51.750 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:51.751 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:51.796 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:51.796 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:51.865 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:09:51.866 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:09:51.866 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:09:51.866 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:09:51.867 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:09:51.867 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:09:51.867 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:09:51.867 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:51.867 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:09:51.868 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:51.868 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:09:51.868 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:09:51.868 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:09:51.869 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:09:51.869 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:09:51.869 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:09:51.869 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:09:51.870 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:09:51.870 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:09:51.870 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:51.870 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:51.871 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:51.871 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:09:51.871 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:09:51.871 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:09:51.872 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:51.872 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:09:51.872 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:09:51.872 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:09:51.873 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:09:51.873 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:09:51.873 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:51.873 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:51.873 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:51.874 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:09:51.874 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:09:51.874 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:09:51.874 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:51.875 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:09:51.875 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:09:51.875 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:09:51.875 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:09:51.876 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:09:51.876 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:51.876 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:51.876 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:51.876 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:09:51.877 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:09:51.877 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:09:51.877 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:51.877 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:09:51.878 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:09:51.878 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:09:51.878 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:09:51.878 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:09:51.879 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:51.879 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:51.879 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:09:51.879 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:51.879 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:09:51.880 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:09:51.880 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:09:51.880 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:51.880 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:09:51.882 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:09:51.882 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:09:51.882 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:51.883 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:51.883 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:51.925 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:51.926 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:51.989 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:09:51.990 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:09:51.990 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:09:51.990 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:09:51.990 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:09:51.991 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:09:51.991 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.991 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.991 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:09:51.992 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:51.992 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:09:51.992 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:51.992 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:51.993 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:09:51.993 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:51.993 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:09:51.993 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:51.993 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:09:51.994 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.994 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.994 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.994 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:51.995 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:09:51.995 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:51.995 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:51.995 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:51.996 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:51.996 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:51.996 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:51.996 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:51.996 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.997 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:51.997 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:51.997 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:51.997 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:09:51.998 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:51.998 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:51.998 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:51.998 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:51.998 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:51.999 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:51.999 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:51.999 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:51.999 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:52.000 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:52.000 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:52.000 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:09:52.000 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:52.001 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:52.001 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:52.001 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:52.001 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:52.001 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:52.002 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:52.002 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:52.002 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:52.002 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:52.003 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:09:52.003 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:52.003 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:09:52.003 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:09:52.003 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:09:52.004 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:52.004 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:09:52.005 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:09:52.006 INFO:tasks.rgw_multisite_tests:running rgw multisite tests on '/home/teuthworker/src/git.ceph.com_ceph-c_a3a8018ba3cfe536252b397411bf18b7828c6098/qa/../src/test/rgw/rgw_multi' with args=['tests.py', 'tests_ps.py'] 2021-11-11T21:09:52.052 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-1 2021-11-11T21:09:55.659 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-2 2021-11-11T21:09:59.643 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-3 2021-11-11T21:09:59.735 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:59.736 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:59.737 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:59.793 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:59.794 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:59.844 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:09:59.859 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:09:59.859 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:59.859 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:09:59.859 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:09:59.860 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:59.860 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:59.860 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:59.860 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:09:59.861 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:59.861 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:59.861 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:09:59.861 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:09:59.862 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:09:59.862 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:09:59.862 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664999.708603_49.1", 2021-11-11T21:09:59.863 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:59.708603Z" 2021-11-11T21:09:59.863 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:09:59.863 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:09:59.864 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:09:59.865 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:59.866 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:09:59.866 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:09:59.908 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:09:59.909 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:09:59.979 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:09:59.980 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:09:59.980 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:09:59.981 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:09:59.981 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:09:59.981 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:09:59.981 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:59.982 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:59.982 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:09:59.982 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:59.983 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:09:59.984 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:59.984 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:59.984 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:09:59.985 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:59.985 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:09:59.986 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:59.986 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:09:59.986 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:59.986 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:59.987 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:59.987 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:59.987 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:09:59.987 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:59.988 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:59.988 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:59.988 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:59.988 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:59.988 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:59.989 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:59.989 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:59.989 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:59.989 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:59.990 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:59.990 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:09:59.990 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:59.990 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:59.990 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:09:59.991 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:59.991 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:59.991 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:59.992 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:09:59.992 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:59.992 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:59.992 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:59.992 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:09:59.993 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:09:59.994 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:09:59.994 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:09:59.994 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664995.637058_38.1", 2021-11-11T21:09:59.995 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:09:59.995 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:09:59.995 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:09:59.995 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:55.637058Z", 2021-11-11T21:09:59.996 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:09:59.996 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:59.996 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:59.996 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:09:59.997 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:09:59.997 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:09:59.997 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:09:59.997 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:09:59.998 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:09:59.998 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:09:59.999 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=3 master=1_1636664999.708603_49.1 target=1_1636664995.637058_38.1 2021-11-11T21:10:05.001 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:05.001 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:05.002 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:05.051 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:05.051 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:05.125 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:05.126 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:05.126 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:05.126 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:05.126 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:05.127 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:05.127 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.127 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.127 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:05.127 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.128 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:05.128 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.128 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.128 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:05.129 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.129 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:05.129 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.129 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:05.129 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.130 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.130 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.130 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.130 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:05.131 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.131 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.131 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:05.131 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.131 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:05.132 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.132 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:05.132 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.133 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.133 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.133 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.133 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:05.134 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.134 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.134 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:05.134 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.134 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:05.135 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.135 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:05.135 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.135 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.136 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.136 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.136 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:05.136 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.136 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.137 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664999.708603_49.1", 2021-11-11T21:10:05.137 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.137 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:05.137 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.137 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:59.708603Z", 2021-11-11T21:10:05.138 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.138 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.138 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.138 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:05.139 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.139 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:05.139 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:05.139 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:05.140 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.140 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:05.141 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:10:05.142 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:10:05.142 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:05.143 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:05.145 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:05.195 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:05.196 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:05.266 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:10:05.267 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:10:05.267 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:10:05.267 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:10:05.268 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:10:05.268 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:05.268 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:05.268 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:05.269 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:10:05.269 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:05.269 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:10:05.269 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:05.270 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:05.270 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:05.270 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:05.270 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:10:05.270 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:05.271 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:05.271 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:05.271 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:05.271 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:05.272 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:05.272 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:10:05.272 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:05.272 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:05.273 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:05.273 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:05.273 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:05.273 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:05.273 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:05.274 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:05.274 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:05.274 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:05.274 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:05.275 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:10:05.275 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:05.275 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:05.275 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:05.275 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:05.276 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:05.276 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:05.276 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:05.277 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:05.277 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:05.277 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:05.277 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:05.278 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:10:05.279 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:05.279 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:05.279 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664999.708603_49.1", 2021-11-11T21:10:05.280 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:05.282 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:05.282 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:05.283 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:59.708603Z", 2021-11-11T21:10:05.283 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:05.283 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:05.283 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:05.284 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:10:05.285 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:05.285 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:10:05.285 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:10:05.285 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:10:05.286 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:05.286 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:10:05.287 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:10:05.288 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:10:05.288 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:05.289 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:05.289 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:05.335 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:05.335 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:05.401 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:05.401 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:05.402 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:05.402 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:05.403 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:05.403 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:05.404 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.405 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.405 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:05.406 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.407 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:05.408 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.408 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.408 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:05.408 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.409 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:05.409 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.409 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:05.409 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.410 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.410 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.410 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.410 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:05.411 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.411 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.411 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:05.411 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.411 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:05.412 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.412 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:05.412 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.412 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.413 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.413 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.413 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:05.414 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.414 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.414 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:05.414 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.414 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:05.415 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.415 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:05.415 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.415 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.416 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.416 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.416 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:05.416 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.417 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.417 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664999.708603_49.1", 2021-11-11T21:10:05.417 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.417 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:05.418 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.418 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:59.708603Z", 2021-11-11T21:10:05.418 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.418 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.419 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.419 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:05.419 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.419 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:05.420 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:05.420 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:05.420 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.420 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:05.421 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:10:05.456 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_create ... ok 2021-11-11T21:10:05.458 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-4 2021-11-11T21:10:05.479 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-5 2021-11-11T21:10:05.510 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-6 2021-11-11T21:10:05.556 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:05.557 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:05.557 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:05.601 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:05.601 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:05.647 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:10:05.665 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:10:05.665 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:05.665 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:05.665 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:10:05.666 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:05.666 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:05.666 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:05.667 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:10:05.667 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:05.667 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:05.667 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:05.668 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:10:05.668 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:05.668 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:05.668 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665005.537490_62.1", 2021-11-11T21:10:05.668 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:05.537490Z" 2021-11-11T21:10:05.669 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:05.669 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:10:05.670 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:10:05.671 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:05.671 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:05.672 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:05.714 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:05.715 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:05.787 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:05.787 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:05.788 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:05.788 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:05.788 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:05.788 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:05.789 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.789 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.789 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:05.789 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.790 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:05.790 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.790 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.790 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:05.791 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.791 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:05.791 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.791 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:05.791 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.792 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.792 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.792 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.792 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:05.793 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.793 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.793 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:05.793 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.793 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:05.794 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.794 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:05.794 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.794 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.795 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.795 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.795 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:05.795 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.796 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.796 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:05.796 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.797 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:05.797 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.798 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:05.798 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.798 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.798 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.799 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:05.799 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:05.799 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:05.799 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:05.799 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665005.468677_50.1", 2021-11-11T21:10:05.800 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:05.800 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:05.800 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:05.801 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:05.468677Z", 2021-11-11T21:10:05.801 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:05.801 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.801 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.801 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:05.802 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:05.802 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:05.802 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:05.803 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:05.803 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:05.803 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:05.803 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=3 master=1_1636665005.537490_62.1 target=1_1636665005.468677_50.1 2021-11-11T21:10:10.805 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:10.806 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:10.806 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:10.857 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:10.857 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:10.922 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:10.962 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:10.962 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:10.962 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:10.963 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:10.963 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:10.963 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:10.963 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:10.964 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:10.964 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:10.964 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:10.964 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:10.965 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:10.965 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:10.965 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:10.965 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:10.966 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:10.966 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:10.966 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:10.966 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:10.967 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:10.967 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:10.967 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:10.967 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:10.967 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:10.968 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:10.968 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:10.968 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:10.968 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:10.969 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:10.969 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:10.969 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:10.969 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:10.970 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:10.970 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:10.970 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:10.970 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:10.971 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:10.971 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:10.971 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:10.971 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:10.971 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:10.972 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:10.972 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:10.972 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:10.972 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:10.973 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:10.973 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:10.973 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:10.974 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665005.537490_62.1", 2021-11-11T21:10:10.974 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:10.974 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:10.974 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:10.975 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:05.537490Z", 2021-11-11T21:10:10.975 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:10.975 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:10.975 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:10.976 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:10.976 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:10.976 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:10.976 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:10.977 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:10.977 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:10.977 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:10.982 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:10:10.983 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:10:10.983 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:10.984 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:10.985 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:11.034 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:11.034 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:11.114 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:10:11.114 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:10:11.114 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:10:11.114 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:10:11.115 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:10:11.115 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:11.115 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:11.115 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.116 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:10:11.116 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.116 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:10:11.116 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:11.117 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:11.117 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:11.117 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:11.117 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:10:11.117 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:11.118 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:11.118 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:11.118 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.118 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.119 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.119 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:10:11.119 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:11.119 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:11.119 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:11.120 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:11.120 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:11.120 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:11.120 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:11.121 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:11.121 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.121 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.121 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.121 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:10:11.122 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:11.122 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:11.122 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:11.122 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:11.123 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:11.123 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:11.123 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:11.123 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:11.123 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.124 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.124 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.124 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:10:11.124 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:11.125 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:11.125 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665005.537490_62.1", 2021-11-11T21:10:11.125 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:11.125 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:11.125 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:11.126 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:05.537490Z", 2021-11-11T21:10:11.126 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:11.126 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.126 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.127 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:10:11.127 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.127 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:10:11.127 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:10:11.127 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:10:11.128 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.128 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:10:11.129 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:10:11.130 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:10:11.130 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:11.131 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:11.131 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:11.176 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:11.176 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:11.239 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:11.240 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:11.240 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:11.240 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:11.241 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:11.241 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:11.241 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:11.241 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:11.241 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:11.242 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:11.242 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:11.242 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:11.242 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:11.243 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:11.243 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:11.243 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:11.243 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:11.244 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:11.244 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:11.244 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.244 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:11.244 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:11.245 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:11.245 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:11.245 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:11.245 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:11.246 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:11.246 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:11.246 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:11.246 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:11.246 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:11.248 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.248 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:11.248 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:11.248 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:11.249 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:11.249 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:11.249 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:11.250 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:11.250 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:11.250 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:11.251 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:11.251 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:11.251 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.251 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:11.251 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:11.252 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:11.252 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:11.252 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:11.252 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665005.537490_62.1", 2021-11-11T21:10:11.253 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:11.253 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:11.253 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:11.254 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:05.537490Z", 2021-11-11T21:10:11.254 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:11.254 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.254 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.255 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:11.255 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:11.255 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:11.255 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:11.255 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:11.256 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.256 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:11.257 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:10:11.583 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:11.584 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:11.585 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:11.627 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:11.628 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:11.686 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:10:11.711 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:10:11.711 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.711 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:11.711 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:10:11.712 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.712 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.712 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:11.712 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:10:11.713 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.714 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.714 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:11.714 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:10:11.714 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.715 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.715 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665005.537490_62.1", 2021-11-11T21:10:11.715 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:05.537490Z" 2021-11-11T21:10:11.715 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.715 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:10:11.716 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:10:11.716 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:11.717 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:11.717 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:11.760 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:11.761 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:11.834 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:11.834 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:11.835 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:11.835 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:11.835 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:11.835 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:11.836 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:11.836 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:11.836 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:11.836 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:11.836 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:11.837 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:11.837 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:11.837 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:11.837 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:11.838 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:11.838 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:11.838 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:11.838 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:11.839 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.839 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:11.839 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:11.839 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:11.839 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:11.840 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:11.840 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:11.840 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:11.840 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:11.841 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:11.841 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:11.841 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:11.841 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.841 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:11.842 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:11.842 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:11.842 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:11.842 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:11.843 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:11.843 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:11.843 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:11.843 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:11.844 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:11.844 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:11.844 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.844 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:11.845 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:11.845 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:11.845 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:11.845 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:11.846 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665005.537490_62.1", 2021-11-11T21:10:11.846 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:11.846 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:11.846 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:11.847 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:05.537490Z", 2021-11-11T21:10:11.847 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:11.847 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.847 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.848 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:11.848 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:11.848 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:11.848 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:11.848 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:11.849 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:11.849 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:11.850 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:10:11.851 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:10:11.851 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:11.852 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:11.852 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:11.895 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:11.896 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:11.963 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:10:11.963 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:10:11.963 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:10:11.964 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:10:11.964 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:10:11.964 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:11.964 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:11.965 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.965 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:10:11.965 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.965 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:10:11.966 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:11.966 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:11.966 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:11.966 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:11.967 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:10:11.967 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:11.967 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:11.967 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:11.968 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.968 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.968 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.968 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:10:11.969 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:11.969 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:11.969 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:11.969 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:11.970 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:11.970 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:11.970 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:11.970 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:11.971 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.971 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.971 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.971 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:10:11.972 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:11.972 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:11.972 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:11.972 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:11.972 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:11.973 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:11.973 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:11.973 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:11.973 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.974 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.974 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:11.974 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:10:11.974 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:11.975 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:11.975 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665005.537490_62.1", 2021-11-11T21:10:11.975 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:11.975 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:11.975 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:11.976 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:05.537490Z", 2021-11-11T21:10:11.976 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:11.976 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.976 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.977 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:10:11.977 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:11.977 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:10:11.977 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:10:11.977 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:10:11.978 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:11.978 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:10:11.979 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:10:11.980 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:10:11.980 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:11.981 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:11.982 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:12.024 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:12.024 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:12.099 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:12.100 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:12.100 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:12.100 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:12.100 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:12.101 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:12.101 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:12.101 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:12.101 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:12.102 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:12.102 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:12.102 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:12.102 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:12.102 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:12.103 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:12.103 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:12.103 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:12.103 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:12.104 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:12.104 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.104 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:12.105 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:12.105 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:12.105 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:12.106 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:12.106 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:12.106 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:12.106 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:12.106 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:12.107 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:12.107 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:12.107 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.107 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:12.108 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:12.108 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:12.108 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:12.108 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:12.109 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:12.109 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:12.109 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:12.109 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:12.109 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:12.110 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:12.110 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.110 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:12.110 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:12.111 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:12.111 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:12.111 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:12.111 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665005.537490_62.1", 2021-11-11T21:10:12.112 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:12.112 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:12.112 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:12.112 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:05.537490Z", 2021-11-11T21:10:12.112 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:12.113 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.113 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.113 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:12.114 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:12.114 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:12.114 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:12.114 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:12.114 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.115 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:12.116 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:10:12.150 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_recreate ... ok 2021-11-11T21:10:12.151 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-7 2021-11-11T21:10:12.169 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-8 2021-11-11T21:10:12.211 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-9 2021-11-11T21:10:12.247 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:12.248 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:12.249 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:12.289 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:12.290 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:12.339 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:10:12.363 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:10:12.363 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:12.363 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:12.364 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:10:12.364 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:12.364 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:12.364 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:12.364 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:10:12.365 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:12.365 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:12.365 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:12.365 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:10:12.366 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:12.366 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:12.366 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665012.234008_80.1", 2021-11-11T21:10:12.366 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:12.234008Z" 2021-11-11T21:10:12.366 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:12.367 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:10:12.368 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:10:12.369 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:12.369 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:12.370 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:12.412 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:12.412 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:12.473 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:12.473 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:12.474 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:12.474 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:12.475 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:12.475 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:12.475 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:12.475 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:12.476 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:12.476 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:12.476 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:12.476 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:12.476 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:12.477 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:12.477 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:12.477 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:12.477 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:12.478 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:12.478 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:12.478 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.478 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:12.479 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:12.479 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:12.479 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:12.479 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:12.480 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:12.480 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:12.480 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:12.480 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:12.481 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:12.481 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:12.481 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.481 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:12.481 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:12.482 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:12.482 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:12.482 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:12.482 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:12.483 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:12.483 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:12.483 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:12.484 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:12.485 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:12.485 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.485 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:12.485 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:12.485 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:12.486 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:12.486 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:12.486 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665012.160352_69.1", 2021-11-11T21:10:12.486 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:12.487 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:12.487 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:12.487 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:12.160352Z", 2021-11-11T21:10:12.488 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:12.488 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.488 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.488 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:12.488 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:12.489 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:12.489 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:12.490 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:12.490 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:12.490 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:12.491 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=3 master=1_1636665012.234008_80.1 target=1_1636665012.160352_69.1 2021-11-11T21:10:17.492 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:17.493 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:17.494 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:17.542 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:17.543 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:17.608 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:17.608 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:17.608 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:17.609 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:17.609 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:17.609 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:17.609 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:17.610 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:17.610 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:17.610 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:17.610 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:17.611 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:17.611 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:17.611 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:17.611 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:17.611 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:17.612 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:17.612 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:17.612 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:17.612 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.613 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:17.613 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:17.613 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:17.613 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:17.613 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:17.614 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:17.614 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:17.614 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:17.614 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:17.615 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:17.615 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:17.615 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.615 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:17.615 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:17.616 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:17.616 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:17.616 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:17.616 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:17.617 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:17.617 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:17.617 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:17.617 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:17.618 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:17.618 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.618 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:17.618 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:17.618 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:17.619 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:17.619 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:17.619 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665012.234008_80.1", 2021-11-11T21:10:17.619 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:17.620 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:17.620 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:17.620 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:12.234008Z", 2021-11-11T21:10:17.620 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:17.620 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.621 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.621 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:17.621 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:17.621 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:17.621 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:17.622 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:17.622 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.622 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:17.623 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:10:17.624 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:10:17.624 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:17.625 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:17.625 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:17.676 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:17.676 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:17.758 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:10:17.758 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:10:17.759 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:10:17.759 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:10:17.759 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:10:17.759 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:17.760 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:17.760 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:17.760 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:10:17.760 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:17.761 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:10:17.761 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:17.761 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:17.761 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:17.761 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:17.762 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:10:17.762 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:17.762 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:17.762 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:17.763 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:17.763 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:17.763 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:17.763 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:10:17.764 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:17.764 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:17.764 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:17.764 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:17.764 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:17.765 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:17.765 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:17.765 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:17.765 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:17.765 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:17.766 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:17.766 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:10:17.766 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:17.766 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:17.766 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:17.767 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:17.767 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:17.767 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:17.767 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:17.767 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:17.768 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:17.768 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:17.768 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:17.768 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:10:17.768 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:17.769 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:17.769 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665012.234008_80.1", 2021-11-11T21:10:17.769 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:17.769 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:17.769 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:17.770 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:12.234008Z", 2021-11-11T21:10:17.770 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:17.770 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:17.770 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:17.771 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:10:17.771 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:17.771 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:10:17.771 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:10:17.771 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:10:17.772 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:17.772 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:10:17.773 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:10:17.774 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:10:17.774 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:17.774 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:17.775 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:17.817 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:17.817 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:17.883 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:17.883 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:17.883 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:17.884 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:17.884 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:17.884 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:17.884 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:17.885 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:17.885 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:17.885 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:17.885 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:17.885 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:17.886 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:17.886 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:17.886 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:17.886 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:17.887 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:17.887 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:17.887 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:17.887 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.888 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:17.888 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:17.888 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:17.888 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:17.888 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:17.889 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:17.889 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:17.889 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:17.890 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:17.890 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:17.890 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:17.890 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.891 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:17.891 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:17.891 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:17.891 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:17.891 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:17.892 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:17.892 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:17.892 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:17.892 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:17.893 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:17.893 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:17.893 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.893 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:17.893 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:17.894 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:17.894 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:17.894 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:17.894 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665012.234008_80.1", 2021-11-11T21:10:17.895 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:17.895 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:17.895 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:17.895 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:12.234008Z", 2021-11-11T21:10:17.895 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:17.896 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.896 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.896 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:17.896 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:17.897 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:17.897 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:17.897 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:17.897 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:17.898 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:17.899 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:10:18.100 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:18.100 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:18.101 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:18.142 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:18.142 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:18.202 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:10:18.217 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:10:18.217 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:18.218 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:18.218 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:10:18.218 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:18.219 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:18.219 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:18.219 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:10:18.219 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:18.219 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:18.220 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:18.220 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:10:18.220 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:18.220 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:18.221 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:18.221 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:10:18.221 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:18.221 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:10:18.222 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:10:18.223 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:18.223 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:18.224 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:18.265 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:18.266 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:18.341 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:18.341 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:18.342 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:18.342 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:18.342 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:18.343 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:18.343 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:18.343 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:18.343 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:18.344 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:18.344 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:18.344 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:18.344 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:18.345 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:18.345 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:18.345 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:18.345 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:18.346 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:18.346 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:18.346 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:18.346 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:18.347 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:18.347 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:18.347 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:18.347 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:18.348 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:18.348 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:18.348 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:18.348 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:18.349 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:18.349 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:18.349 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:18.349 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:18.350 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:18.350 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:18.350 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:18.350 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:18.351 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:18.351 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:18.351 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:18.351 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:18.352 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:18.352 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:18.352 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:18.352 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:18.353 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:18.353 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:18.353 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:18.353 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:18.354 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665012.234008_80.1", 2021-11-11T21:10:18.354 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:18.354 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:18.354 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:18.355 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:12.234008Z", 2021-11-11T21:10:18.355 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:18.355 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:18.355 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:18.355 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:18.356 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:18.356 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:18.356 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:18.356 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:18.357 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:18.357 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:18.358 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=3 master=1_1636665018.067753_86.1 target=1_1636665012.234008_80.1 2021-11-11T21:10:23.359 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:23.360 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:23.361 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:23.409 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:23.410 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:23.484 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:23.484 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:23.485 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:23.485 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:23.485 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:23.485 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:23.485 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:23.486 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:23.486 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:23.486 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:23.486 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:23.487 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:23.487 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:23.487 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:23.487 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:23.488 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:23.488 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:23.488 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:23.489 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:23.489 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.489 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:23.489 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:23.490 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:23.490 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:23.490 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:23.490 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:23.490 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:23.491 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:23.491 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:23.491 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:23.491 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:23.492 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.492 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:23.492 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:23.492 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:23.493 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:23.493 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:23.493 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:23.493 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:23.494 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:23.494 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:23.494 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:23.494 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:23.495 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.495 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:23.495 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:23.495 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:23.496 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:23.496 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:23.496 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:23.496 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:23.496 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:23.497 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:23.497 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:23.497 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:23.497 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.497 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.498 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:23.498 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:23.498 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:23.498 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:23.499 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:23.499 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.499 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:23.504 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:10:23.505 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:10:23.505 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:23.506 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:23.507 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:23.557 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:23.557 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:23.639 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:10:23.640 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:10:23.640 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:10:23.640 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:10:23.640 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:10:23.641 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:23.641 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:23.641 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:23.642 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:10:23.642 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:23.642 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:10:23.642 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:23.642 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:23.643 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:23.643 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:23.643 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:10:23.643 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:23.644 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:23.644 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:23.644 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:23.644 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:23.644 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:23.645 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:10:23.645 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:23.645 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:23.645 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:23.646 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:23.646 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:23.646 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:23.646 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:23.646 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:23.647 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:23.647 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:23.647 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:23.647 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:10:23.648 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:23.648 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:23.648 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:23.648 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:23.648 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:23.649 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:23.649 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:23.649 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:23.649 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:23.649 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:23.650 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:23.650 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:10:23.650 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:23.650 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:23.651 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:23.651 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:23.651 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:23.651 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:23.651 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:23.652 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:23.652 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:23.652 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:23.652 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:10:23.652 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:23.653 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:10:23.653 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:10:23.653 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:10:23.653 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:23.653 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:10:23.655 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:10:23.655 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:10:23.656 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:23.656 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:23.657 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:23.700 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:23.701 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:23.782 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:23.783 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:23.783 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:23.783 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:23.783 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:23.784 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:23.784 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:23.784 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:23.784 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:23.785 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:23.785 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:23.785 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:23.785 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:23.786 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:23.786 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:23.786 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:23.786 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:23.786 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:23.787 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:23.787 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.787 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:23.787 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:23.788 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:23.788 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:23.788 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:23.788 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:23.789 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:23.789 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:23.789 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:23.789 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:23.789 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:23.790 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.790 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:23.790 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:23.790 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:23.791 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:23.791 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:23.791 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:23.791 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:23.792 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:23.792 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:23.792 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:23.792 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:23.792 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.793 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:23.793 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:23.793 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:23.793 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:23.794 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:23.794 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:23.794 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:23.795 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:23.795 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:23.795 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:23.795 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:23.796 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.796 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.796 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:23.796 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:23.797 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:23.797 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:23.797 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:23.798 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:23.798 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:23.799 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:10:23.809 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_remove ... ok 2021-11-11T21:10:23.810 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-10 2021-11-11T21:10:23.841 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-11 2021-11-11T21:10:23.882 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-12 2021-11-11T21:10:23.954 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:10:23.955+0000 7f04fcc36700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:23.955 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:10:23.955+0000 7fcacda57700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:23.956 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:10:23.956+0000 7f0505c48700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:23.956 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:10:23.954+0000 7fb6c822c700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:23.957 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:10:23.955+0000 7fb6c8a2d700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:23.957 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:10:23.956+0000 7fcac4a45700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:24.154 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:10:24.153+0000 7fb6c9a2f700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:24.154 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:10:24.153+0000 7fb6c922e700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:24.154 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:10:24.153+0000 7fb6c6228700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:24.155 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:10:24.155+0000 7fcabfa3b700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:24.155 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:10:24.155+0000 7fcace258700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:24.156 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:10:24.155+0000 7fcaca250700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:24.156 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:10:24.155+0000 7f0506449700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:24.156 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:10:24.155+0000 7f0500c3e700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:24.157 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:10:24.155+0000 7f0506449700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:24.547 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:24.548 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:24.548 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:24.621 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:24.622 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:24.746 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:10:24.779 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:10:24.780 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:24.780 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:24.780 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:10:24.780 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:24.780 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:24.781 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:24.781 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:10:24.781 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:24.781 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:24.782 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665023.901260_78.1", 2021-11-11T21:10:24.782 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:23.901260Z" 2021-11-11T21:10:24.782 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:24.782 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:24.783 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:24.783 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:10:24.783 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:24.783 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:10:24.784 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:10:24.785 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:24.785 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:24.786 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:24.856 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:24.856 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:25.039 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:25.039 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:25.039 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:25.039 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:25.040 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:25.040 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:25.040 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:25.041 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:25.041 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:25.041 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:25.042 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:25.042 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:25.042 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:25.042 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:25.043 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:25.043 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:25.043 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:25.043 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:25.044 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:25.044 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:25.044 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:25.044 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:25.044 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:25.045 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:25.045 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:25.045 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:25.045 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:25.046 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:25.046 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:25.046 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:25.046 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:25.046 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:25.047 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:25.047 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:25.047 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:25.047 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:25.047 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:25.048 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665023.829391_68.1", 2021-11-11T21:10:25.048 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:25.048 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:25.049 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:25.049 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:23.829391Z", 2021-11-11T21:10:25.049 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:25.050 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:25.050 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:25.050 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:25.051 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:25.051 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:25.051 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:25.051 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:25.052 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:25.052 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:25.052 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:25.052 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:25.052 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:25.053 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:25.053 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:25.053 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:25.054 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:25.054 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:25.054 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:25.055 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:25.055 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:25.055 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:25.056 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665023.901260_78.1 target=1_1636665023.829391_68.1 2021-11-11T21:10:30.057 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:30.058 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:30.059 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:30.103 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:30.103 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:30.169 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:30.170 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:30.170 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:30.170 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:30.171 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:30.171 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:30.171 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:30.172 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:30.172 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:30.172 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:30.173 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:30.173 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:30.174 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:30.174 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:30.174 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:30.174 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:30.175 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:30.175 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:30.175 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:30.175 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.176 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:30.176 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:30.176 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:30.176 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:30.176 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:30.177 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:30.177 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:30.177 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:30.177 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:30.178 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:30.178 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:30.178 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.178 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:30.179 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:30.179 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:30.179 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:30.179 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:30.179 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665023.901260_78.1", 2021-11-11T21:10:30.180 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:30.180 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:30.180 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:30.180 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:23.901260Z", 2021-11-11T21:10:30.181 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:30.181 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.181 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:30.181 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:30.182 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:30.182 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:30.182 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:30.183 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:30.183 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:30.183 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:30.183 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:30.184 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:30.184 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:30.184 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.185 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.185 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:30.185 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:30.185 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:30.186 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:30.186 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:30.186 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.186 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:30.187 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:10:30.187 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:10:30.188 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:30.188 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:30.189 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:30.234 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:30.234 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:30.311 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:10:30.312 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:10:30.312 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:10:30.313 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:10:30.313 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:10:30.313 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:30.313 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:30.314 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:30.314 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:10:30.314 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:30.314 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:10:30.315 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:30.315 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:30.315 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:30.315 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:30.315 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:10:30.316 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:30.316 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:30.316 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:30.316 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:30.317 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:30.317 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:30.317 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:10:30.317 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:30.318 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:30.318 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:30.318 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:30.318 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:30.318 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:30.319 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:30.319 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:30.319 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:30.319 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:30.320 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:30.320 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:10:30.320 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:30.320 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:30.320 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665023.901260_78.1", 2021-11-11T21:10:30.321 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:30.321 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:30.321 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:30.321 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:23.901260Z", 2021-11-11T21:10:30.322 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:30.322 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:30.322 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:30.322 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:30.323 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:10:30.323 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:30.323 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:30.323 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:30.324 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:30.324 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:30.324 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:30.324 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:30.324 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:30.325 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:30.325 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:30.325 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:10:30.325 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:30.326 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:10:30.326 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:10:30.326 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:10:30.326 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:30.326 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:10:30.328 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:10:30.328 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:10:30.329 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:30.329 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:30.330 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:30.372 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:30.372 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:30.438 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:30.439 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:30.439 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:30.439 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:30.440 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:30.440 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:30.440 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:30.440 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:30.441 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:30.441 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:30.441 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:30.441 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:30.441 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:30.442 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:30.442 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:30.442 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:30.442 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:30.443 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:30.443 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:30.443 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.443 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:30.443 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:30.444 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:30.444 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:30.444 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:30.444 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:30.445 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:30.445 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:30.445 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:30.445 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:30.446 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:30.446 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.446 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:30.446 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:30.446 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:30.447 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:30.447 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:30.447 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665023.901260_78.1", 2021-11-11T21:10:30.448 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:30.448 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:30.448 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:30.449 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:23.901260Z", 2021-11-11T21:10:30.449 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:30.450 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.450 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:30.450 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:30.450 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:30.451 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:30.451 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:30.451 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:30.451 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:30.451 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:30.452 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:30.452 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:30.452 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:30.452 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.453 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.453 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:30.453 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:30.453 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:30.454 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:30.454 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:30.454 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:30.454 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:30.456 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:10:30.456 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:30.457 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:30.457 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-10 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:30.545 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:30.546 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:30.615 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:30.616+0000 7f70872d9380 1 bucket sync caught up with source: 2021-11-11T21:10:30.615 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000001.4.5, , 00000000001.7.5, , , , 00000000001.5.5, 00000000001.7.5, , ] 2021-11-11T21:10:30.616 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000001.4.5, , 00000000001.7.5, , , , 00000000001.5.5, 00000000001.7.5, , ] 2021-11-11T21:10:30.616 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:30.616+0000 7f70872d9380 0 bucket checkpoint complete 2021-11-11T21:10:30.631 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-10 zones={test-zone2, test-zone1} 2021-11-11T21:10:30.640 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:30.647 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:30.647 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:30.648 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:30.648 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:30.648 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:30.653 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:30.654 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:30.654 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:30.655 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:30.664 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:30.721 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:30.833 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:30.936 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:30.992 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:31.097 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:31.195 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:31.253 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:31.362 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:31.460 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:31.522 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:31.634 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:31.736 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-10 zones={test-zone2, test-zone1} 2021-11-11T21:10:31.737 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:31.737 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:31.738 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-10 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:31.798 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:31.798 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:31.860 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:31.859+0000 7f25af4f2380 1 bucket sync caught up with source: 2021-11-11T21:10:31.860 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000001.4.5, , 00000000001.7.5, , , , 00000000001.5.5, 00000000001.7.5, , ] 2021-11-11T21:10:31.860 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000001.4.5, , 00000000001.7.5, , , , 00000000001.5.5, 00000000001.7.5, , ] 2021-11-11T21:10:31.861 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:31.859+0000 7f25af4f2380 0 bucket checkpoint complete 2021-11-11T21:10:31.880 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-10 zones={test-zone3, test-zone1} 2021-11-11T21:10:31.888 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:31.892 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:31.893 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:31.893 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:31.894 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:31.894 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:31.899 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:31.899 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:31.900 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:31.900 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:31.908 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:31.964 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:32.073 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:32.171 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:32.228 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:32.337 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:32.435 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:32.492 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:32.597 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:32.699 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:32.760 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:32.865 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:32.963 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-10 zones={test-zone3, test-zone1} 2021-11-11T21:10:32.964 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:32.964 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-10', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:32.965 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-10 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:33.020 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:33.021 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:33.094 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:33.095+0000 7f1e6b797380 1 bucket sync caught up with source: 2021-11-11T21:10:33.095 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000001.4.5, , 00000000001.7.5, , , , 00000000001.5.5, 00000000001.7.5, , ] 2021-11-11T21:10:33.095 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000001.4.5, , 00000000001.7.5, , , , 00000000001.5.5, 00000000001.7.5, , ] 2021-11-11T21:10:33.095 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:33.095+0000 7f1e6b797380 0 bucket checkpoint complete 2021-11-11T21:10:33.111 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-11', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:33.112 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-11', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:33.112 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-11 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:33.161 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:33.162 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:33.225 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:33.226+0000 7f2f98537380 1 bucket sync caught up with source: 2021-11-11T21:10:33.226 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000001.12.5, , 00000000001.63.5, , , , 00000000001.74.5, 00000000001.13.5, , ] 2021-11-11T21:10:33.226 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000001.12.5, , 00000000001.63.5, , , , 00000000001.74.5, 00000000001.13.5, , ] 2021-11-11T21:10:33.226 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:33.226+0000 7f2f98537380 0 bucket checkpoint complete 2021-11-11T21:10:33.240 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-11 zones={test-zone1, test-zone2} 2021-11-11T21:10:33.248 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:33.253 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:33.253 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:33.254 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:33.254 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:33.255 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:33.259 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:33.260 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:33.260 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:33.261 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:33.270 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:33.324 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:33.433 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:33.531 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:33.590 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:33.698 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:33.799 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:33.857 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:33.970 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:34.075 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:34.133 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:34.247 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:34.347 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-11 zones={test-zone1, test-zone2} 2021-11-11T21:10:34.348 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-11', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:34.349 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-11', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:34.349 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-11 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:34.406 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:34.406 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:34.477 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:34.477+0000 7f670a264380 1 bucket sync caught up with source: 2021-11-11T21:10:34.477 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000001.12.5, , 00000000001.63.5, , , , 00000000001.74.5, 00000000001.13.5, , ] 2021-11-11T21:10:34.477 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000001.12.5, , 00000000001.63.5, , , , 00000000001.74.5, 00000000001.13.5, , ] 2021-11-11T21:10:34.478 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:34.477+0000 7f670a264380 0 bucket checkpoint complete 2021-11-11T21:10:34.494 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-11 zones={test-zone3, test-zone2} 2021-11-11T21:10:34.503 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:34.508 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:34.508 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:34.509 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:34.509 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:34.510 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:34.515 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:34.515 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:34.515 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:34.516 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:34.525 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:34.580 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:34.686 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:34.792 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:34.852 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:34.958 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:35.055 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:35.116 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:35.222 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:35.323 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:35.381 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:35.494 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:35.604 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-11 zones={test-zone3, test-zone2} 2021-11-11T21:10:35.605 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-12', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:35.606 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-12', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:35.606 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-12 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:35.662 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:35.663 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:35.737 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:35.738+0000 7fdd9df79380 1 bucket sync caught up with source: 2021-11-11T21:10:35.737 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000001.27.5, , 00000000001.126.5, , , , 00000000001.42.5, 00000000001.74.5, , ] 2021-11-11T21:10:35.738 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000001.27.5, , 00000000001.126.5, , , , 00000000001.42.5, 00000000001.74.5, , ] 2021-11-11T21:10:35.738 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:35.738+0000 7fdd9df79380 0 bucket checkpoint complete 2021-11-11T21:10:35.756 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-12 zones={test-zone1, test-zone3} 2021-11-11T21:10:35.764 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:35.769 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:35.770 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:35.770 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:35.771 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:35.771 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:35.776 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:35.776 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:35.777 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:35.777 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:35.786 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:35.845 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:35.949 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:36.051 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:36.108 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:36.218 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:36.315 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:36.373 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:36.482 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:36.582 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:36.636 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:36.741 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:36.844 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-12 zones={test-zone1, test-zone3} 2021-11-11T21:10:36.845 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-12', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:36.846 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-12', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:36.847 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-12 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:36.902 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:36.903 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:36.967 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:36.968+0000 7f77c9cb5380 1 bucket sync caught up with source: 2021-11-11T21:10:36.967 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000001.27.5, , 00000000001.126.5, , , , 00000000001.42.5, 00000000001.74.5, , ] 2021-11-11T21:10:36.968 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000001.27.5, , 00000000001.126.5, , , , 00000000001.42.5, 00000000001.74.5, , ] 2021-11-11T21:10:36.968 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:36.968+0000 7f77c9cb5380 0 bucket checkpoint complete 2021-11-11T21:10:36.982 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-12 zones={test-zone2, test-zone3} 2021-11-11T21:10:36.990 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:36.995 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:36.996 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:36.996 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:36.997 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:36.997 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:37.002 DEBUG:tasks.rgw_multi.zone_rados:o=& 2021-11-11T21:10:37.002 DEBUG:tasks.rgw_multi.zone_rados:o=: 2021-11-11T21:10:37.003 DEBUG:tasks.rgw_multi.zone_rados:o=_myobj 2021-11-11T21:10:37.003 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:37.011 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:37.069 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:37.177 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=& 2021-11-11T21:10:37.280 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:37.336 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:37.445 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=: 2021-11-11T21:10:37.547 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:37.610 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:37.722 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=_myobj 2021-11-11T21:10:37.827 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:37.885 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:37.994 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:38.095 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-12 zones={test-zone2, test-zone3} 2021-11-11T21:10:38.096 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_object_sync ... ok 2021-11-11T21:10:38.097 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-13 2021-11-11T21:10:38.117 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-14 2021-11-11T21:10:38.154 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-15 2021-11-11T21:10:38.339 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:38.340 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:38.340 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:38.360 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:10:38.361+0000 7fb6cd236700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:38.361 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:10:38.361+0000 7f04f9c30700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:38.361 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:10:38.361+0000 7fcac2240700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:38.384 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:38.384 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:38.407 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:10:38.407+0000 7fb6c9a2f700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:38.420 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:10:38.420+0000 7fcaca250700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:10:38.450 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:10:38.471 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:10:38.472 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:38.472 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:38.472 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:10:38.472 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:38.473 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:38.473 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:38.473 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:10:38.473 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:38.474 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:38.474 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665038.178540_93.1", 2021-11-11T21:10:38.474 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:38.178540Z" 2021-11-11T21:10:38.474 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:38.474 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:38.475 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:38.475 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:10:38.475 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:38.475 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:10:38.477 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:10:38.477 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:38.478 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:38.478 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:38.523 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:38.524 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:38.690 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:38.690 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:38.690 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:38.691 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:38.691 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:38.691 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:38.691 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:38.692 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:38.692 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:38.692 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:38.692 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:38.693 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:38.693 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:38.693 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:38.693 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:38.694 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:38.694 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:38.694 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:38.694 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:38.694 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:38.695 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:38.695 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:38.695 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:38.695 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:38.696 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:38.696 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:38.696 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:38.696 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:38.697 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:38.697 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:38.697 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:38.697 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:38.698 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:38.698 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:38.698 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:38.698 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:38.699 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:38.699 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665038.107435_82.1", 2021-11-11T21:10:38.699 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:38.699 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:38.699 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:38.700 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:38.107435Z", 2021-11-11T21:10:38.700 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:38.700 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:38.700 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:38.701 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:38.701 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:38.701 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:38.701 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:38.702 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:38.702 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:38.702 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:38.702 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:38.702 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:38.703 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:38.703 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:38.703 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:38.703 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:38.704 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:38.704 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:38.704 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:38.704 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:38.705 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:38.705 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:38.705 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665038.178540_93.1 target=1_1636665038.107435_82.1 2021-11-11T21:10:43.707 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:43.708 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:43.709 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:43.750 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:43.751 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:43.833 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:43.833 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:43.834 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:43.834 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:43.834 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:43.834 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:43.835 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:43.835 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:43.835 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:43.835 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:43.836 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:43.836 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:43.836 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:43.836 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:43.837 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:43.837 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:43.837 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:43.837 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:43.838 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:43.838 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:43.838 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:43.838 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:43.838 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:43.839 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:43.839 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:43.839 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:43.839 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:43.840 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:43.840 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:43.840 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:43.840 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:43.841 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:43.841 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:43.841 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:43.841 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:43.841 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:43.842 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:43.842 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665038.178540_93.1", 2021-11-11T21:10:43.842 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:43.842 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:43.843 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:43.843 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:38.178540Z", 2021-11-11T21:10:43.843 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:43.843 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:43.843 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:43.844 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:43.844 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:43.844 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:43.844 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:43.845 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:43.845 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:43.845 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:43.845 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:43.846 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:43.846 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:43.846 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:43.846 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:43.846 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:43.847 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:43.847 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:43.847 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:43.847 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:43.848 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:43.848 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:43.849 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:10:43.850 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:10:43.850 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:43.851 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:43.851 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:43.897 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:43.897 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:43.977 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:10:43.978 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:10:43.978 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:10:43.978 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:10:43.978 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:10:43.979 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:43.979 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:43.979 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:43.979 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:10:43.979 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:43.980 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:10:43.980 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:43.980 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:43.980 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:43.981 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:43.981 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:10:43.981 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:43.981 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:43.981 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:43.982 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:43.982 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:43.982 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:43.982 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:10:43.983 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:43.983 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:43.983 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:43.983 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:43.983 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:43.984 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:43.984 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:43.984 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:43.984 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:43.985 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:43.985 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:43.985 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:10:43.985 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:43.985 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:43.986 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665038.107435_82.1", 2021-11-11T21:10:43.986 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:43.986 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:43.986 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:43.987 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:38.107435Z", 2021-11-11T21:10:43.987 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:43.987 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:43.987 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:43.988 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:43.988 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:10:43.988 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:43.988 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:43.988 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:43.989 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:43.989 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:43.989 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:43.989 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:43.990 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:43.990 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:43.990 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:43.990 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:10:43.990 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:43.991 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:10:43.991 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:10:43.991 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:10:43.991 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:43.992 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:10:43.993 WARNING:rgw_multi.tests:zone test-zone3 behind master: shard=2 master=1_1636665038.178540_93.1 target=1_1636665038.107435_82.1 2021-11-11T21:10:48.994 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:48.995 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:48.996 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:49.046 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:49.047 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:49.131 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:10:49.132 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:10:49.132 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:10:49.132 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:10:49.132 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:10:49.133 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:49.133 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:49.133 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:49.134 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:10:49.134 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:49.134 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:10:49.135 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:49.135 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:49.135 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:49.136 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:49.136 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:10:49.136 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:49.137 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:49.137 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:49.137 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:49.138 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:49.138 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:49.138 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:10:49.138 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:49.139 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:49.139 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:49.139 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:49.139 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:49.139 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:49.140 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:10:49.140 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:49.140 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:49.140 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:49.141 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:49.141 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:10:49.141 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:49.142 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:49.142 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665038.178540_93.1", 2021-11-11T21:10:49.142 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:49.142 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:49.143 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:49.143 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:38.178540Z", 2021-11-11T21:10:49.143 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:49.143 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:49.144 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:49.144 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:49.144 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:10:49.144 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:10:49.144 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:10:49.145 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:49.145 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:10:49.145 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:10:49.145 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:10:49.146 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:49.146 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:10:49.146 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:49.146 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:49.147 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:10:49.147 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:49.147 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:10:49.147 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:10:49.147 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:10:49.148 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:49.148 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:10:49.149 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:10:49.150 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:10:49.150 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:49.151 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:49.152 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:49.200 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:49.200 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:49.276 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:49.276 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:49.277 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:49.277 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:49.277 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:49.277 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:49.277 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:49.278 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:49.278 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:49.278 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:49.278 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:49.279 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:49.279 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:49.279 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:49.279 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:49.280 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:49.280 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:49.280 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:49.280 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:49.280 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:49.281 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:49.281 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:49.281 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:49.281 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:49.282 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:49.282 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:49.282 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:49.282 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:49.283 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:49.283 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:49.283 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:49.283 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:49.283 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:49.284 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:49.284 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:49.284 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:49.284 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:49.285 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665038.178540_93.1", 2021-11-11T21:10:49.285 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:49.285 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:49.285 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:49.285 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:38.178540Z", 2021-11-11T21:10:49.286 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:49.286 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:49.286 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:49.286 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:49.286 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:49.287 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:49.287 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:49.287 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:49.287 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:49.288 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:49.288 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:49.288 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:49.288 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:49.288 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:49.289 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:49.289 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:49.289 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:49.289 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:49.289 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:49.290 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:49.290 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:49.290 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:49.291 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:10:49.292 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:49.292 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:49.293 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:49.341 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:49.342 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:49.405 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:49.406+0000 7fa719613380 1 bucket sync caught up with source: 2021-11-11T21:10:49.405 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2021-11-11T21:10:49.406 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000001.7.5, , , , , , , , , ] 2021-11-11T21:10:49.406 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:49.406+0000 7fa719613380 0 bucket checkpoint complete 2021-11-11T21:10:49.420 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-13 zones={test-zone2, test-zone1} 2021-11-11T21:10:49.429 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:49.475 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:49.476 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:49.481 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:49.531 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:49.592 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:49.703 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:49.806 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-13 zones={test-zone2, test-zone1} 2021-11-11T21:10:49.806 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:49.807 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:49.808 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:49.861 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:49.861 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:49.937 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:49.938+0000 7f039e9b5380 1 bucket sync caught up with source: 2021-11-11T21:10:49.938 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2021-11-11T21:10:49.938 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000001.7.5, , , , , , , , , ] 2021-11-11T21:10:49.938 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:49.938+0000 7f039e9b5380 0 bucket checkpoint complete 2021-11-11T21:10:49.952 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-13 zones={test-zone3, test-zone1} 2021-11-11T21:10:49.960 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:50.007 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:50.007 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:50.012 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:50.063 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:50.116 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:50.225 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:50.323 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-13 zones={test-zone3, test-zone1} 2021-11-11T21:10:50.324 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:50.324 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:50.325 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:50.378 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:50.379 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:50.452 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:50.452+0000 7fb046923380 1 bucket sync caught up with source: 2021-11-11T21:10:50.452 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000001.7.5, , , , , , , , , ] 2021-11-11T21:10:50.452 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000001.7.5, , , , , , , , , ] 2021-11-11T21:10:50.453 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:50.452+0000 7fb046923380 0 bucket checkpoint complete 2021-11-11T21:10:50.469 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:50.470 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:50.471 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-14 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:50.519 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:50.520 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:50.592 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:50.592+0000 7f6a1b503380 1 bucket sync caught up with source: 2021-11-11T21:10:50.592 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000001.64.5, , , , , , , , , ] 2021-11-11T21:10:50.592 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000001.64.5, , , , , , , , , ] 2021-11-11T21:10:50.593 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:50.592+0000 7f6a1b503380 0 bucket checkpoint complete 2021-11-11T21:10:50.610 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-14 zones={test-zone1, test-zone2} 2021-11-11T21:10:50.619 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:50.667 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:50.668 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:50.673 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:50.723 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:50.784 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:50.890 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:50.988 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-14 zones={test-zone1, test-zone2} 2021-11-11T21:10:50.988 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:50.989 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:50.990 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-14 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:51.039 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:51.039 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:51.116 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:51.117+0000 7f99facce380 1 bucket sync caught up with source: 2021-11-11T21:10:51.117 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000001.64.5, , , , , , , , , ] 2021-11-11T21:10:51.117 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000001.64.5, , , , , , , , , ] 2021-11-11T21:10:51.117 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:51.117+0000 7f99facce380 0 bucket checkpoint complete 2021-11-11T21:10:51.130 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-14 zones={test-zone3, test-zone2} 2021-11-11T21:10:51.139 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:51.187 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:51.187 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:51.193 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:51.244 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:51.301 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:51.406 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:51.504 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-14 zones={test-zone3, test-zone2} 2021-11-11T21:10:51.505 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:51.505 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:51.506 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-15 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:51.555 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:51.556 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:51.616 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:51.616+0000 7fcc857f3380 1 bucket sync caught up with source: 2021-11-11T21:10:51.616 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000001.64.5, , , , , , , , , ] 2021-11-11T21:10:51.616 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000001.64.5, , , , , , , , , ] 2021-11-11T21:10:51.617 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:51.616+0000 7fcc857f3380 0 bucket checkpoint complete 2021-11-11T21:10:51.631 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-15 zones={test-zone1, test-zone3} 2021-11-11T21:10:51.638 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:51.691 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:51.692 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:51.697 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:51.747 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:51.800 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:51.906 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:52.003 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-15 zones={test-zone1, test-zone3} 2021-11-11T21:10:52.003 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:52.004 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:52.004 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-15 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:52.059 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:52.059 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:52.121 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:52.121+0000 7fc16c301380 1 bucket sync caught up with source: 2021-11-11T21:10:52.121 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000001.64.5, , , , , , , , , ] 2021-11-11T21:10:52.121 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000001.64.5, , , , , , , , , ] 2021-11-11T21:10:52.122 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:52.121+0000 7fc16c301380 0 bucket checkpoint complete 2021-11-11T21:10:52.136 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-15 zones={test-zone2, test-zone3} 2021-11-11T21:10:52.144 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:52.191 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:52.192 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:52.196 DEBUG:tasks.rgw_multi.zone_rados:o=myobj 2021-11-11T21:10:52.247 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:52.300 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:52.409 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=myobj 2021-11-11T21:10:52.511 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-15 zones={test-zone2, test-zone3} 2021-11-11T21:10:52.524 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:52.524 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:52.525 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:52.577 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:52.577 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:52.637 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:52.638+0000 7f03575ce380 1 bucket sync caught up with source: 2021-11-11T21:10:52.637 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000002.9.5, , , , , , , , , ] 2021-11-11T21:10:52.637 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000002.9.5, , , , , , , , , ] 2021-11-11T21:10:52.638 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:52.638+0000 7f03575ce380 0 bucket checkpoint complete 2021-11-11T21:10:52.652 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-13 zones={test-zone2, test-zone1} 2021-11-11T21:10:52.661 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:52.707 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:52.763 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-13 zones={test-zone2, test-zone1} 2021-11-11T21:10:52.763 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:52.764 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:52.764 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:52.812 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:52.813 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:52.889 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:52.890+0000 7fee69e98380 1 bucket sync caught up with source: 2021-11-11T21:10:52.889 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000002.9.5, , , , , , , , , ] 2021-11-11T21:10:52.890 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000002.9.5, , , , , , , , , ] 2021-11-11T21:10:52.890 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:52.890+0000 7fee69e98380 0 bucket checkpoint complete 2021-11-11T21:10:52.906 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-13 zones={test-zone3, test-zone1} 2021-11-11T21:10:52.914 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:52.963 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:53.023 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-13 zones={test-zone3, test-zone1} 2021-11-11T21:10:53.023 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:53.024 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-13', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:53.024 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-13 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:53.074 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:53.074 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:53.131 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:53.132+0000 7fe99d9ef380 1 bucket sync caught up with source: 2021-11-11T21:10:53.131 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000002.9.5, , , , , , , , , ] 2021-11-11T21:10:53.132 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000002.9.5, , , , , , , , , ] 2021-11-11T21:10:53.132 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:53.132+0000 7fe99d9ef380 0 bucket checkpoint complete 2021-11-11T21:10:53.158 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:53.159 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:53.159 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-14 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:53.208 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:53.208 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:53.268 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:53.269+0000 7ff6cc374380 1 bucket sync caught up with source: 2021-11-11T21:10:53.268 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000002.146.5, , , , , , , , , ] 2021-11-11T21:10:53.268 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000002.146.5, , , , , , , , , ] 2021-11-11T21:10:53.269 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:53.269+0000 7ff6cc374380 0 bucket checkpoint complete 2021-11-11T21:10:53.284 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-14 zones={test-zone1, test-zone2} 2021-11-11T21:10:53.292 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:53.339 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:53.391 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-14 zones={test-zone1, test-zone2} 2021-11-11T21:10:53.391 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:53.392 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-14', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:53.392 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-14 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:53.445 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:53.445 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:53.508 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:53.509+0000 7f1c9b8c3380 1 bucket sync caught up with source: 2021-11-11T21:10:53.508 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000002.146.5, , , , , , , , , ] 2021-11-11T21:10:53.509 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000002.146.5, , , , , , , , , ] 2021-11-11T21:10:53.509 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:53.509+0000 7f1c9b8c3380 0 bucket checkpoint complete 2021-11-11T21:10:53.531 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-14 zones={test-zone3, test-zone2} 2021-11-11T21:10:53.539 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:53.587 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:53.644 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-14 zones={test-zone3, test-zone2} 2021-11-11T21:10:53.659 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:53.660 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:53.660 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-15 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:53.709 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:53.710 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:53.790 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:53.791+0000 7f766bce9380 1 waiting for incremental sync to catch up: 2021-11-11T21:10:53.790 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000001.64.5, , , , , , , , , ] 2021-11-11T21:10:53.791 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000002.126.5, , , , , , , , , ] 2021-11-11T21:10:58.792 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:58.792+0000 7f766bce9380 1 bucket sync caught up with source: 2021-11-11T21:10:58.792 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000002.126.5, , , , , , , , , ] 2021-11-11T21:10:58.792 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000002.126.5, , , , , , , , , ] 2021-11-11T21:10:58.793 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:10:58.792+0000 7f766bce9380 0 bucket checkpoint complete 2021-11-11T21:10:58.811 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-15 zones={test-zone1, test-zone3} 2021-11-11T21:10:58.820 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:58.867 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:58.923 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-15 zones={test-zone1, test-zone3} 2021-11-11T21:10:58.924 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:58.924 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-15', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:10:58.925 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-15 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:10:58.980 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:58.981 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:59.133 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:59.133+0000 7f9b1e279380 1 bucket sync caught up with source: 2021-11-11T21:10:59.133 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000002.126.5, , , , , , , , , ] 2021-11-11T21:10:59.133 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000002.126.5, , , , , , , , , ] 2021-11-11T21:10:59.133 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:10:59.133+0000 7f9b1e279380 0 bucket checkpoint complete 2021-11-11T21:10:59.149 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-15 zones={test-zone2, test-zone3} 2021-11-11T21:10:59.157 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:10:59.203 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:10:59.260 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-15 zones={test-zone2, test-zone3} 2021-11-11T21:10:59.260 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_object_delete ... ok 2021-11-11T21:10:59.261 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-16 2021-11-11T21:10:59.282 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-17 2021-11-11T21:10:59.319 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-18 2021-11-11T21:10:59.407 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:59.408 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:59.409 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:59.450 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:59.451 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:59.512 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:10:59.527 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:10:59.528 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:59.528 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:59.529 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:10:59.529 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:59.529 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:59.530 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:10:59.530 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:10:59.531 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:59.531 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:59.532 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665059.402172_111.1", 2021-11-11T21:10:59.532 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:59.402172Z" 2021-11-11T21:10:59.533 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:10:59.533 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:10:59.534 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:59.534 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:10:59.535 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:10:59.535 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:10:59.537 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:10:59.538 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:59.539 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:10:59.540 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:10:59.590 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:10:59.590 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:10:59.677 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:10:59.678 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:10:59.678 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:10:59.679 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:10:59.679 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:10:59.679 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:10:59.679 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:59.680 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:59.680 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:10:59.680 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:59.680 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:10:59.681 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:59.681 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:59.681 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:10:59.682 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:59.682 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:10:59.682 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:59.683 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:10:59.683 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:59.683 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:59.684 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:59.684 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:59.684 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:10:59.684 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:59.685 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:59.685 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:10:59.685 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:59.685 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:59.686 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:59.686 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:10:59.686 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:59.686 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:59.687 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:59.687 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:59.687 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:10:59.687 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:59.688 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:59.688 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665059.271988_94.1", 2021-11-11T21:10:59.688 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:59.688 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:59.688 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:59.689 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:59.271988Z", 2021-11-11T21:10:59.689 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:59.689 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:59.689 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:59.690 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:10:59.690 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:10:59.690 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:10:59.690 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:10:59.690 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:10:59.691 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:10:59.691 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:10:59.691 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:10:59.691 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:10:59.692 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:10:59.692 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:59.692 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:59.692 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:10:59.692 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:10:59.693 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:10:59.693 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:10:59.693 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:10:59.693 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:10:59.694 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:10:59.695 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665059.402172_111.1 target=1_1636665059.271988_94.1 2021-11-11T21:11:04.697 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:11:04.698 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:11:04.699 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:11:04.750 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:04.750 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:04.836 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:11:04.836 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:11:04.837 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:11:04.837 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:11:04.837 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:11:04.837 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:11:04.838 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:11:04.838 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:11:04.838 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:11:04.838 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:11:04.839 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:11:04.839 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:11:04.839 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:11:04.839 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:11:04.840 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:11:04.840 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:11:04.840 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:11:04.840 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:11:04.840 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:11:04.841 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:04.841 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:11:04.841 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:11:04.842 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:11:04.842 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:11:04.842 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:11:04.842 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:11:04.843 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:11:04.843 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:11:04.843 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:11:04.844 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:11:04.844 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:11:04.844 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:04.844 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:11:04.845 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:11:04.845 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:11:04.845 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:11:04.845 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:11:04.846 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665059.402172_111.1", 2021-11-11T21:11:04.846 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:11:04.846 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:11:04.846 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:11:04.847 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:59.402172Z", 2021-11-11T21:11:04.847 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:11:04.847 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:04.848 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:11:04.848 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:11:04.848 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:11:04.848 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:11:04.849 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:11:04.849 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:11:04.850 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:11:04.850 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:11:04.850 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:11:04.850 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:11:04.851 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:11:04.851 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:04.851 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:04.852 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:11:04.852 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:11:04.852 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:11:04.852 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:11:04.852 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:11:04.853 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:04.853 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:11:04.854 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:11:04.855 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:11:04.855 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:11:04.856 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:11:04.856 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:11:04.905 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:04.906 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:04.974 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:11:04.974 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:11:04.974 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:11:04.975 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:11:04.975 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:11:04.975 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:11:04.975 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:11:04.976 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:11:04.976 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:11:04.976 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:11:04.977 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:11:04.977 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:11:04.977 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:11:04.977 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:11:04.977 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:11:04.978 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:11:04.978 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:11:04.978 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:11:04.978 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:11:04.979 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:11:04.979 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:11:04.979 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:11:04.979 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:11:04.980 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:11:04.980 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:11:04.980 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:11:04.980 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:11:04.981 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:11:04.981 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:11:04.981 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:11:04.981 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:11:04.982 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:11:04.982 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:11:04.982 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:11:04.982 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:11:04.983 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:11:04.983 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:11:04.983 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665059.402172_111.1", 2021-11-11T21:11:04.983 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:11:04.983 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:11:04.984 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:11:04.984 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:59.402172Z", 2021-11-11T21:11:04.984 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:11:04.984 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:11:04.985 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:11:04.985 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:11:04.985 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:11:04.985 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:11:04.986 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:11:04.986 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:11:04.986 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:11:04.986 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:11:04.987 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:11:04.987 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:11:04.987 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:11:04.987 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:11:04.988 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:11:04.988 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:11:04.988 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:11:04.988 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:11:04.989 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:11:04.989 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:11:04.989 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:11:04.989 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:11:04.990 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:11:04.991 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:11:04.991 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:11:04.992 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:11:04.993 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:11:05.036 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:05.036 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:05.095 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:11:05.095 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:11:05.095 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:11:05.096 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:11:05.096 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:11:05.096 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:11:05.096 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:11:05.097 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:11:05.097 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:11:05.097 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:11:05.097 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:11:05.097 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:11:05.098 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:11:05.098 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:11:05.098 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:11:05.098 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:11:05.099 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:11:05.099 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:11:05.099 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:11:05.099 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:05.100 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:11:05.100 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:11:05.100 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:11:05.101 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:11:05.101 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:11:05.101 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:11:05.102 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:11:05.102 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:11:05.102 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:11:05.103 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:11:05.103 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:11:05.104 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:05.104 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:11:05.104 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:11:05.105 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:11:05.105 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:11:05.106 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:11:05.106 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665059.402172_111.1", 2021-11-11T21:11:05.107 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:11:05.107 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:11:05.108 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:11:05.108 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:59.402172Z", 2021-11-11T21:11:05.108 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:11:05.109 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:05.109 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:11:05.110 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:11:05.110 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:11:05.111 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:11:05.111 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:11:05.112 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:11:05.112 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:11:05.113 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:11:05.113 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:11:05.113 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:11:05.114 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:11:05.114 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:05.115 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:05.115 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:11:05.116 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:11:05.116 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:11:05.117 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:11:05.117 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:11:05.118 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:11:05.118 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:11:05.121 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:11:05.175 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:05.176 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:05.176 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:05.226 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:05.227 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:05.287 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:05.287+0000 7f3d51092380 1 waiting for incremental sync to catch up: 2021-11-11T21:11:05.287 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , , , ] 2021-11-11T21:11:05.287 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000002.9.10, , , , , ] 2021-11-11T21:11:10.289 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:10.289+0000 7f3d51092380 1 bucket sync caught up with source: 2021-11-11T21:11:10.289 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , 00000000006.17.9, , , , , ] 2021-11-11T21:11:10.289 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000002.9.10, , , , , ] 2021-11-11T21:11:10.290 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:10.289+0000 7f3d51092380 0 bucket checkpoint complete 2021-11-11T21:11:10.308 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:10.309 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:10.309 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:10.364 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:10.365 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:10.437 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:10.437+0000 7f227f2dc380 1 bucket sync caught up with source: 2021-11-11T21:11:10.438 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000006.17.9, , , , , ] 2021-11-11T21:11:10.438 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000006.17.9, , , , , ] 2021-11-11T21:11:10.438 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:10.437+0000 7f227f2dc380 0 bucket checkpoint complete 2021-11-11T21:11:10.455 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:10.456 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:10.456 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:10.512 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:10.512 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:10.593 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:10.594+0000 7f1013073380 1 bucket sync caught up with source: 2021-11-11T21:11:10.594 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , 00000000006.17.9, , , , , ] 2021-11-11T21:11:10.594 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000006.17.9, , , , , ] 2021-11-11T21:11:10.594 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:10.594+0000 7f1013073380 0 bucket checkpoint complete 2021-11-11T21:11:10.663 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:10.663 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:10.664 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:10.714 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:10.714 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:10.786 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:10.786+0000 7f0f01c9b380 1 waiting for incremental sync to catch up: 2021-11-11T21:11:10.786 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , , , ] 2021-11-11T21:11:10.787 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000002.200.10, , , , , ] 2021-11-11T21:11:15.788 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:15.788+0000 7f0f01c9b380 1 bucket sync caught up with source: 2021-11-11T21:11:15.789 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000006.208.9, , , , , ] 2021-11-11T21:11:15.790 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000002.200.10, , , , , ] 2021-11-11T21:11:15.790 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:15.788+0000 7f0f01c9b380 0 bucket checkpoint complete 2021-11-11T21:11:15.808 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:15.809 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:15.810 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:15.868 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:15.869 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:15.944 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:15.944+0000 7f068a886380 1 bucket sync caught up with source: 2021-11-11T21:11:15.944 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000006.208.9, , , , , ] 2021-11-11T21:11:15.945 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000006.208.9, , , , , ] 2021-11-11T21:11:15.945 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:15.944+0000 7f068a886380 0 bucket checkpoint complete 2021-11-11T21:11:16.011 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:16.012 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:16.012 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:16.062 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:16.063 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:16.125 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:16.126+0000 7f2483ff5380 1 waiting for incremental sync to catch up: 2021-11-11T21:11:16.125 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , , , ] 2021-11-11T21:11:16.126 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000002.247.10, , , , , ] 2021-11-11T21:11:21.127 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:21.127+0000 7f2483ff5380 1 bucket sync caught up with source: 2021-11-11T21:11:21.127 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000006.255.9, , , , , ] 2021-11-11T21:11:21.128 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000002.247.10, , , , , ] 2021-11-11T21:11:21.128 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:21.127+0000 7f2483ff5380 0 bucket checkpoint complete 2021-11-11T21:11:21.147 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:21.147 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:21.148 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:21.206 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:21.207 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:21.271 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:21.271+0000 7f4c6adfc380 1 bucket sync caught up with source: 2021-11-11T21:11:21.271 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , 00000000006.255.9, , , , , ] 2021-11-11T21:11:21.271 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000006.255.9, , , , , ] 2021-11-11T21:11:21.272 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:21.271+0000 7f4c6adfc380 0 bucket checkpoint complete 2021-11-11T21:11:22.314 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:11:22.314+0000 7fb6b95f5700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-5) 2021-11-11T21:11:22.383 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:11:22.383+0000 7fcab3609700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-5) 2021-11-11T21:11:23.018 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.019 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.019 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:23.076 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:23.076 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:23.134 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:23.135+0000 7f8e85898380 1 bucket sync caught up with source: 2021-11-11T21:11:23.135 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000010.37.5, , 00000000003.15.9, , 00000000010.25.9, , , , , ] 2021-11-11T21:11:23.135 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000010.37.5, , 00000000003.15.9, , 00000000010.25.9, , , , , ] 2021-11-11T21:11:23.135 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:23.135+0000 7f8e85898380 0 bucket checkpoint complete 2021-11-11T21:11:23.149 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.150 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.150 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:23.210 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:23.211 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:23.279 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:23.279+0000 7f7d9cc9d380 1 bucket sync caught up with source: 2021-11-11T21:11:23.279 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000010.37.5, , 00000000003.15.9, , 00000000010.25.9, , , , , ] 2021-11-11T21:11:23.279 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000010.37.5, , 00000000003.15.9, , 00000000010.25.9, , , , , ] 2021-11-11T21:11:23.279 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:23.279+0000 7f7d9cc9d380 0 bucket checkpoint complete 2021-11-11T21:11:23.294 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.295 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.295 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:23.345 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:23.346 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:23.414 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:23.414+0000 7f409b1aa380 1 bucket sync caught up with source: 2021-11-11T21:11:23.414 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000010.37.5, , 00000000003.15.9, , 00000000010.25.9, , , , , ] 2021-11-11T21:11:23.415 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000010.37.5, , 00000000003.15.9, , 00000000010.25.9, , , , , ] 2021-11-11T21:11:23.415 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:23.414+0000 7f409b1aa380 0 bucket checkpoint complete 2021-11-11T21:11:23.431 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.432 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.432 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:23.482 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:23.483 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:23.549 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:23.550+0000 7fce944b0380 1 bucket sync caught up with source: 2021-11-11T21:11:23.550 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.248.9, , 00000000010.214.5, , 00000000007.185.9, , , , , ] 2021-11-11T21:11:23.550 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.248.9, , 00000000010.214.5, , 00000000007.185.9, , , , , ] 2021-11-11T21:11:23.550 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:23.550+0000 7fce944b0380 0 bucket checkpoint complete 2021-11-11T21:11:23.565 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.565 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.566 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:23.615 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:23.616 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:23.675 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:23.676+0000 7ff5aa974380 1 bucket sync caught up with source: 2021-11-11T21:11:23.676 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.248.9, , 00000000010.214.5, , 00000000007.185.9, , , , , ] 2021-11-11T21:11:23.676 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.248.9, , 00000000010.214.5, , 00000000007.185.9, , , , , ] 2021-11-11T21:11:23.676 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:23.676+0000 7ff5aa974380 0 bucket checkpoint complete 2021-11-11T21:11:23.697 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.698 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.699 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:23.747 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:23.748 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:23.822 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:23.823+0000 7fedf7cbc380 1 bucket sync caught up with source: 2021-11-11T21:11:23.822 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.261.9, , 00000000003.201.9, , 00000000014.208.5, , , , , ] 2021-11-11T21:11:23.823 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.261.9, , 00000000003.201.9, , 00000000014.208.5, , , , , ] 2021-11-11T21:11:23.823 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:23.823+0000 7fedf7cbc380 0 bucket checkpoint complete 2021-11-11T21:11:23.843 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.843 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:23.844 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:23.892 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:23.893 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:23.959 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:23.959+0000 7f49eb0b5380 1 bucket sync caught up with source: 2021-11-11T21:11:23.959 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000003.261.9, , 00000000003.201.9, , 00000000014.208.5, , , , , ] 2021-11-11T21:11:23.959 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000003.261.9, , 00000000003.201.9, , 00000000014.208.5, , , , , ] 2021-11-11T21:11:23.960 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:23.959+0000 7f49eb0b5380 0 bucket checkpoint complete 2021-11-11T21:11:23.977 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-16 zones={test-zone2, test-zone1} 2021-11-11T21:11:23.989 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:23.996 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:23.997 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:23.997 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:23.998 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:23.999 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:24.004 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:24.004 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:24.005 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:24.005 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:24.015 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:24.028 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:24.096 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:24.156 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:24.213 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:24.325 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:24.424 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:24.481 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:24.592 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:24.696 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:24.754 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:24.867 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:24.972 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-16 zones={test-zone2, test-zone1} 2021-11-11T21:11:24.973 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-16 zones={test-zone3, test-zone1} 2021-11-11T21:11:24.983 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:24.990 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:24.991 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:24.991 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:24.992 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:24.992 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:24.997 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:24.998 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:24.999 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:24.999 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:25.010 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:25.025 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:25.091 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:25.152 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:25.213 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:25.327 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:25.424 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:25.484 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:25.596 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:25.700 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:25.763 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:25.877 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:25.989 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-16 zones={test-zone3, test-zone1} 2021-11-11T21:11:25.989 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-16 zones={test-zone3, test-zone2} 2021-11-11T21:11:25.999 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:26.010 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:26.011 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:26.011 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:26.012 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:26.012 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:26.059 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:26.059 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:26.060 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:26.060 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:26.115 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:26.132 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:26.205 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:26.269 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:26.330 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:26.445 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:26.548 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:26.606 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:26.718 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:26.825 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:26.882 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:27.002 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:27.104 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-16 zones={test-zone3, test-zone2} 2021-11-11T21:11:27.105 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.105 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.106 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:27.160 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:27.161 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:27.236 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:27.236+0000 7f5b37e69380 1 bucket sync caught up with source: 2021-11-11T21:11:27.236 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000010.22.5, , 00000000003.14.9, , 00000000007.21.9, , , , , ] 2021-11-11T21:11:27.236 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000010.22.5, , 00000000003.14.9, , 00000000007.21.9, , , , , ] 2021-11-11T21:11:27.236 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:27.236+0000 7f5b37e69380 0 bucket checkpoint complete 2021-11-11T21:11:27.253 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.253 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.254 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:27.306 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:27.306 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:27.378 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:27.379+0000 7fe971188380 1 bucket sync caught up with source: 2021-11-11T21:11:27.379 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000010.22.5, , 00000000003.14.9, , 00000000007.21.9, , , , , ] 2021-11-11T21:11:27.379 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000010.22.5, , 00000000003.14.9, , 00000000007.21.9, , , , , ] 2021-11-11T21:11:27.379 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:27.379+0000 7fe971188380 0 bucket checkpoint complete 2021-11-11T21:11:27.398 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.399 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.400 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:27.453 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:27.453 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:27.516 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:27.517+0000 7fe54179e380 1 bucket sync caught up with source: 2021-11-11T21:11:27.517 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000010.22.5, , 00000000003.14.9, , 00000000007.21.9, , , , , ] 2021-11-11T21:11:27.517 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000010.22.5, , 00000000003.14.9, , 00000000007.21.9, , , , , ] 2021-11-11T21:11:27.517 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:27.517+0000 7fe54179e380 0 bucket checkpoint complete 2021-11-11T21:11:27.532 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.532 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.533 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:27.587 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:27.587 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:27.654 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:27.654+0000 7f08d7181380 1 bucket sync caught up with source: 2021-11-11T21:11:27.654 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.136.9, , 00000000010.252.5, , 00000000009.214.9, , , , , ] 2021-11-11T21:11:27.655 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.136.9, , 00000000010.252.5, , 00000000009.214.9, , , , , ] 2021-11-11T21:11:27.655 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:27.654+0000 7f08d7181380 0 bucket checkpoint complete 2021-11-11T21:11:27.669 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.670 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.670 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:27.719 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:27.720 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:27.777 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:27.777+0000 7fe0450e7380 1 bucket sync caught up with source: 2021-11-11T21:11:27.777 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.136.9, , 00000000010.252.5, , 00000000009.214.9, , , , , ] 2021-11-11T21:11:27.778 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.136.9, , 00000000010.252.5, , 00000000009.214.9, , , , , ] 2021-11-11T21:11:27.778 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:27.777+0000 7fe0450e7380 0 bucket checkpoint complete 2021-11-11T21:11:27.794 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.794 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.795 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:27.843 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:27.843 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:27.921 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:27.922+0000 7f13ad1ff380 1 bucket sync caught up with source: 2021-11-11T21:11:27.921 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.134.9, , 00000000003.248.9, , 00000000014.218.5, , , , , ] 2021-11-11T21:11:27.922 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.134.9, , 00000000003.248.9, , 00000000014.218.5, , , , , ] 2021-11-11T21:11:27.922 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:27.922+0000 7f13ad1ff380 0 bucket checkpoint complete 2021-11-11T21:11:27.941 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.941 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:27.942 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:27.991 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:27.992 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:28.054 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:28.053+0000 7f0ded8fb380 1 bucket sync caught up with source: 2021-11-11T21:11:28.054 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000003.134.9, , 00000000003.248.9, , 00000000014.218.5, , , , , ] 2021-11-11T21:11:28.054 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000003.134.9, , 00000000003.248.9, , 00000000014.218.5, , , , , ] 2021-11-11T21:11:28.054 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:28.053+0000 7f0ded8fb380 0 bucket checkpoint complete 2021-11-11T21:11:28.068 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-17 zones={test-zone2, test-zone1} 2021-11-11T21:11:28.076 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:28.082 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:28.083 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:28.083 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:28.084 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:28.084 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:28.089 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:28.090 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:28.090 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:28.091 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:28.100 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:28.113 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:28.178 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:28.236 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:28.292 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:28.399 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:28.499 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:28.560 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:28.674 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:28.775 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:28.834 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:28.938 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:29.036 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-17 zones={test-zone2, test-zone1} 2021-11-11T21:11:29.036 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-17 zones={test-zone3, test-zone1} 2021-11-11T21:11:29.044 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:29.049 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:29.050 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:29.050 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:29.051 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:29.051 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:29.056 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:29.057 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:29.057 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:29.058 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:29.067 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:29.079 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:29.143 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:29.199 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:29.256 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:29.364 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:29.467 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:29.525 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:29.639 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:29.740 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:29.797 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:29.906 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:30.003 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-17 zones={test-zone3, test-zone1} 2021-11-11T21:11:30.004 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-17 zones={test-zone3, test-zone2} 2021-11-11T21:11:30.013 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:30.018 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:30.019 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:30.019 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:30.020 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:30.020 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:30.026 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:30.027 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:30.027 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:30.028 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:30.083 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:30.097 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:30.167 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:30.228 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:30.285 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:30.396 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:30.500 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:30.559 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:30.668 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:30.776 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:30.840 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:30.952 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:31.052 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-17 zones={test-zone3, test-zone2} 2021-11-11T21:11:31.053 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:31.054 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:31.054 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:31.110 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:31.111 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:31.177 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:31.177+0000 7fbb4ab0f380 1 bucket sync caught up with source: 2021-11-11T21:11:31.177 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000010.21.5, , 00000000006.38.5, , 00000000007.47.9, , , , , ] 2021-11-11T21:11:31.178 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000010.21.5, , 00000000006.38.5, , 00000000007.47.9, , , , , ] 2021-11-11T21:11:31.178 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:31.177+0000 7fbb4ab0f380 0 bucket checkpoint complete 2021-11-11T21:11:31.192 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:31.193 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:31.193 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:31.243 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:31.244 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:31.301 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:31.302+0000 7fd0b12fe380 1 bucket sync caught up with source: 2021-11-11T21:11:31.302 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000010.21.5, , 00000000006.38.5, , 00000000007.47.9, , , , , ] 2021-11-11T21:11:31.302 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000010.21.5, , 00000000006.38.5, , 00000000007.47.9, , , , , ] 2021-11-11T21:11:31.302 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:31.302+0000 7fd0b12fe380 0 bucket checkpoint complete 2021-11-11T21:11:31.317 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:31.317 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:31.318 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:31.368 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:31.368 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:31.426 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:31.426+0000 7fa91ce7b380 1 bucket sync caught up with source: 2021-11-11T21:11:31.426 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000010.21.5, , 00000000006.38.5, , 00000000007.47.9, , , , , ] 2021-11-11T21:11:31.427 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000010.21.5, , 00000000006.38.5, , 00000000007.47.9, , , , , ] 2021-11-11T21:11:31.427 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:31.426+0000 7fa91ce7b380 0 bucket checkpoint complete 2021-11-11T21:11:31.447 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:31.448 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:31.449 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:31.498 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:31.498 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:31.566 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:31.566+0000 7f341e653380 1 bucket sync caught up with source: 2021-11-11T21:11:31.566 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.65.9, , 00000000010.205.5, , 00000000007.254.9, , , , , ] 2021-11-11T21:11:31.566 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.65.9, , 00000000010.205.5, , 00000000007.254.9, , , , , ] 2021-11-11T21:11:31.567 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:31.566+0000 7f341e653380 0 bucket checkpoint complete 2021-11-11T21:11:31.590 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:31.590 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:31.591 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:31.640 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:31.640 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:31.709 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:31.710+0000 7f4e395f6380 1 waiting for incremental sync to catch up: 2021-11-11T21:11:31.709 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.65.9, , 00000000010.205.5, , 00000000004.242.9, , , , , ] 2021-11-11T21:11:31.710 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.65.9, , 00000000010.205.5, , 00000000007.254.9, , , , , ] 2021-11-11T21:11:36.711 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:36.712+0000 7f4e395f6380 1 waiting for incremental sync to catch up: 2021-11-11T21:11:36.711 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.65.9, , 00000000010.205.5, , 00000000004.242.9, , , , , ] 2021-11-11T21:11:36.712 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.65.9, , 00000000010.205.5, , 00000000007.254.9, , , , , ] 2021-11-11T21:11:41.713 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:41.713+0000 7f4e395f6380 1 waiting for incremental sync to catch up: 2021-11-11T21:11:41.713 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.65.9, , 00000000010.205.5, , 00000000004.242.9, , , , , ] 2021-11-11T21:11:41.713 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.65.9, , 00000000010.205.5, , 00000000007.254.9, , , , , ] 2021-11-11T21:11:46.715 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:46.715+0000 7f4e395f6380 1 bucket sync caught up with source: 2021-11-11T21:11:46.715 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.65.9, , 00000000010.205.5, , 00000000007.254.9, , , , , ] 2021-11-11T21:11:46.715 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.65.9, , 00000000010.205.5, , 00000000007.254.9, , , , , ] 2021-11-11T21:11:46.716 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:46.715+0000 7f4e395f6380 0 bucket checkpoint complete 2021-11-11T21:11:46.734 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:46.735 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:46.735 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:46.788 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:46.789 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:46.869 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:46.869+0000 7f5fd2327380 1 bucket sync caught up with source: 2021-11-11T21:11:46.869 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000003.66.9, , 00000000006.220.5, , 00000000016.281.5, , , , , ] 2021-11-11T21:11:46.869 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000003.66.9, , 00000000006.220.5, , 00000000016.281.5, , , , , ] 2021-11-11T21:11:46.870 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:46.869+0000 7f5fd2327380 0 bucket checkpoint complete 2021-11-11T21:11:46.889 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:46.890 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:46.890 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:46.946 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:46.946 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:47.008 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:47.009+0000 7f96bbb82380 1 bucket sync caught up with source: 2021-11-11T21:11:47.009 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000003.66.9, , 00000000006.220.5, , 00000000016.281.5, , , , , ] 2021-11-11T21:11:47.009 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000003.66.9, , 00000000006.220.5, , 00000000016.281.5, , , , , ] 2021-11-11T21:11:47.009 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:47.009+0000 7f96bbb82380 0 bucket checkpoint complete 2021-11-11T21:11:47.024 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-18 zones={test-zone2, test-zone1} 2021-11-11T21:11:47.033 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:47.039 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:47.039 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:47.040 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:47.040 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:47.040 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:47.045 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:47.046 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:47.046 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:47.047 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:47.056 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:47.068 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:47.134 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:47.192 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:47.248 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:47.355 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:47.458 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:47.513 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:47.624 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:47.723 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:47.781 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:47.893 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:47.991 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-18 zones={test-zone2, test-zone1} 2021-11-11T21:11:47.992 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-18 zones={test-zone3, test-zone1} 2021-11-11T21:11:48.000 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:48.005 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:48.005 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:48.006 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:48.006 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:48.007 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:48.011 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:48.012 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:48.012 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:48.013 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:48.022 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:48.033 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:48.102 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:48.159 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:48.221 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:48.327 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:48.471 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:48.529 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:48.645 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:48.743 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:48.800 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:48.911 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:49.011 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-18 zones={test-zone3, test-zone1} 2021-11-11T21:11:49.012 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-18 zones={test-zone3, test-zone2} 2021-11-11T21:11:49.021 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:49.026 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:49.026 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:49.027 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:49.027 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:49.027 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:49.075 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:49.076 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:49.076 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:49.077 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:49.087 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:49.102 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:49.172 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:49.232 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:49.290 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:49.399 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:49.512 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:49.569 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:49.686 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:49.792 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:49.850 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:49.959 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:50.068 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-18 zones={test-zone3, test-zone2} 2021-11-11T21:11:50.646 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:50.647 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:50.648 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:50.710 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:50.711 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:50.776 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:50.777+0000 7f0d0ed4b380 1 bucket sync caught up with source: 2021-11-11T21:11:50.777 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000011.50.5, , 00000000004.18.5, , 00000000011.41.5, , , , , ] 2021-11-11T21:11:50.777 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000011.50.5, , 00000000004.18.5, , 00000000011.41.5, , , , , ] 2021-11-11T21:11:50.777 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:50.777+0000 7f0d0ed4b380 0 bucket checkpoint complete 2021-11-11T21:11:50.793 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:50.794 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:50.795 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:50.857 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:50.857 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:50.927 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:50.927+0000 7f510899f380 1 bucket sync caught up with source: 2021-11-11T21:11:50.927 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000011.50.5, , 00000000004.18.5, , 00000000011.41.5, , , , , ] 2021-11-11T21:11:50.928 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000011.50.5, , 00000000004.18.5, , 00000000011.41.5, , , , , ] 2021-11-11T21:11:50.928 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:50.927+0000 7f510899f380 0 bucket checkpoint complete 2021-11-11T21:11:50.952 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:50.952 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:50.953 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:51.003 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:51.004 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:51.066 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:51.067+0000 7f9f69248380 1 bucket sync caught up with source: 2021-11-11T21:11:51.067 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000011.50.5, , 00000000004.18.5, , 00000000011.41.5, , , , , ] 2021-11-11T21:11:51.067 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000011.50.5, , 00000000004.18.5, , 00000000011.41.5, , , , , ] 2021-11-11T21:11:51.067 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:51.067+0000 7f9f69248380 0 bucket checkpoint complete 2021-11-11T21:11:51.084 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:51.084 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:51.085 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:51.134 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:51.134 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:51.212 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:51.211+0000 7fca4314e380 1 bucket sync caught up with source: 2021-11-11T21:11:51.213 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000004.257.5, , 00000000011.221.5, , 00000000008.211.5, , , , , ] 2021-11-11T21:11:51.213 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000004.257.5, , 00000000011.221.5, , 00000000008.211.5, , , , , ] 2021-11-11T21:11:51.213 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:51.211+0000 7fca4314e380 0 bucket checkpoint complete 2021-11-11T21:11:51.232 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:51.232 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:51.233 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:51.281 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:51.282 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:51.341 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:51.340+0000 7f01021a0380 1 bucket sync caught up with source: 2021-11-11T21:11:51.341 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000004.257.5, , 00000000011.221.5, , 00000000008.211.5, , , , , ] 2021-11-11T21:11:51.341 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000004.257.5, , 00000000011.221.5, , 00000000008.211.5, , , , , ] 2021-11-11T21:11:51.342 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:51.340+0000 7f01021a0380 0 bucket checkpoint complete 2021-11-11T21:11:51.356 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:51.357 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:51.357 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:51.406 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:51.406 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:51.462 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:51.462+0000 7fd7b9c3f380 1 bucket sync caught up with source: 2021-11-11T21:11:51.463 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000004.284.5, , 00000000004.204.5, , 00000000015.226.5, , , , , ] 2021-11-11T21:11:51.463 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000004.284.5, , 00000000004.204.5, , 00000000015.226.5, , , , , ] 2021-11-11T21:11:51.463 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:51.462+0000 7fd7b9c3f380 0 bucket checkpoint complete 2021-11-11T21:11:51.478 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:51.478 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-16', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:51.479 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-16 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:51.528 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:51.528 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:51.597 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:51.597+0000 7f6bb312f380 1 bucket sync caught up with source: 2021-11-11T21:11:51.597 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000004.284.5, , 00000000004.204.5, , 00000000015.226.5, , , , , ] 2021-11-11T21:11:51.597 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000004.284.5, , 00000000004.204.5, , 00000000015.226.5, , , , , ] 2021-11-11T21:11:51.597 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:51.597+0000 7f6bb312f380 0 bucket checkpoint complete 2021-11-11T21:11:51.614 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-16 zones={test-zone2, test-zone1} 2021-11-11T21:11:51.623 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:51.628 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:51.629 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:51.629 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:51.630 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:51.630 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:51.635 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:51.636 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:51.636 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:51.637 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:51.647 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:51.661 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:51.728 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:51.788 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:51.846 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:51.952 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:52.061 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:52.117 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:52.232 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:52.344 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:52.402 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:52.511 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:52.620 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-16 zones={test-zone2, test-zone1} 2021-11-11T21:11:52.621 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-16 zones={test-zone3, test-zone1} 2021-11-11T21:11:52.629 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:52.634 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:52.635 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:52.635 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:52.636 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:52.636 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:52.641 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:52.641 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:52.642 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:52.642 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:52.652 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:52.665 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:52.731 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:52.792 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:52.850 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:52.956 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:53.055 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:53.116 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:53.227 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:53.339 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:53.397 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:53.502 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:53.604 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-16 zones={test-zone3, test-zone1} 2021-11-11T21:11:53.604 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-16 zones={test-zone3, test-zone2} 2021-11-11T21:11:53.614 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:53.619 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:53.619 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:53.620 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:53.620 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:53.621 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:53.675 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:53.676 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:53.676 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:53.677 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:53.688 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:53.703 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:53.768 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:53.828 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:53.887 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:54.001 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:54.108 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:54.166 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:54.277 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:54.386 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:54.442 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:54.561 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:54.664 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-16 zones={test-zone3, test-zone2} 2021-11-11T21:11:54.665 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:54.666 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:54.667 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:54.723 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:54.724 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:54.785 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:54.785+0000 7f51ada7d380 1 bucket sync caught up with source: 2021-11-11T21:11:54.785 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000011.28.5, , 00000000004.17.5, , 00000000008.24.5, , , , , ] 2021-11-11T21:11:54.785 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000011.28.5, , 00000000004.17.5, , 00000000008.24.5, , , , , ] 2021-11-11T21:11:54.785 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:54.785+0000 7f51ada7d380 0 bucket checkpoint complete 2021-11-11T21:11:54.801 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:54.801 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:54.802 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:54.855 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:54.856 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:54.914 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:54.914+0000 7fbb7eb96380 1 bucket sync caught up with source: 2021-11-11T21:11:54.914 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000011.28.5, , 00000000004.17.5, , 00000000008.24.5, , , , , ] 2021-11-11T21:11:54.914 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000011.28.5, , 00000000004.17.5, , 00000000008.24.5, , , , , ] 2021-11-11T21:11:54.915 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:54.914+0000 7fbb7eb96380 0 bucket checkpoint complete 2021-11-11T21:11:54.929 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:54.930 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:54.931 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:54.980 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:54.981 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:55.054 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:55.055+0000 7fc9b86fd380 1 bucket sync caught up with source: 2021-11-11T21:11:55.055 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000011.28.5, , 00000000004.17.5, , 00000000008.24.5, , , , , ] 2021-11-11T21:11:55.055 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000011.28.5, , 00000000004.17.5, , 00000000008.24.5, , , , , ] 2021-11-11T21:11:55.055 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:55.055+0000 7fc9b86fd380 0 bucket checkpoint complete 2021-11-11T21:11:55.069 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:55.070 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:55.071 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:55.119 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:55.119 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:55.185 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:55.185+0000 7f25b8b3b380 1 bucket sync caught up with source: 2021-11-11T21:11:55.186 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000004.139.5, , 00000000011.259.5, , 00000000010.217.5, , , , , ] 2021-11-11T21:11:55.186 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000004.139.5, , 00000000011.259.5, , 00000000010.217.5, , , , , ] 2021-11-11T21:11:55.186 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:55.185+0000 7f25b8b3b380 0 bucket checkpoint complete 2021-11-11T21:11:55.202 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:55.203 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:55.204 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:55.254 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:55.254 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:55.314 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:55.314+0000 7f462ebaa380 1 bucket sync caught up with source: 2021-11-11T21:11:55.315 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000004.139.5, , 00000000011.259.5, , 00000000010.217.5, , , , , ] 2021-11-11T21:11:55.315 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000004.139.5, , 00000000011.259.5, , 00000000010.217.5, , , , , ] 2021-11-11T21:11:55.315 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:55.314+0000 7f462ebaa380 0 bucket checkpoint complete 2021-11-11T21:11:55.331 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:55.332 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:55.332 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:55.381 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:55.381 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:55.437 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:55.436+0000 7f369fd9e380 1 bucket sync caught up with source: 2021-11-11T21:11:55.437 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000004.137.5, , 00000000004.251.5, , 00000000015.221.5, , , , , ] 2021-11-11T21:11:55.437 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000004.137.5, , 00000000004.251.5, , 00000000015.221.5, , , , , ] 2021-11-11T21:11:55.438 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:55.436+0000 7f369fd9e380 0 bucket checkpoint complete 2021-11-11T21:11:55.452 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:55.453 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-17', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:55.454 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-17 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:55.505 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:55.505 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:55.577 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:55.577+0000 7f62d6440380 1 bucket sync caught up with source: 2021-11-11T21:11:55.577 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000004.137.5, , 00000000004.251.5, , 00000000015.221.5, , , , , ] 2021-11-11T21:11:55.577 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000004.137.5, , 00000000004.251.5, , 00000000015.221.5, , , , , ] 2021-11-11T21:11:55.577 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:55.577+0000 7f62d6440380 0 bucket checkpoint complete 2021-11-11T21:11:55.594 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-17 zones={test-zone2, test-zone1} 2021-11-11T21:11:55.603 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:55.608 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:55.608 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:55.609 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:55.609 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:55.610 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:55.614 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:55.615 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:55.615 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:55.616 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:55.625 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:55.640 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:55.706 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:55.767 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:55.829 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:55.941 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:56.048 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:56.105 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:56.213 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:56.315 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:56.372 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:56.483 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:56.588 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-17 zones={test-zone2, test-zone1} 2021-11-11T21:11:56.588 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-17 zones={test-zone3, test-zone1} 2021-11-11T21:11:56.596 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:56.601 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:56.602 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:56.602 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:56.603 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:56.603 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:56.608 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:56.608 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:56.609 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:56.609 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:56.620 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:56.632 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:56.703 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:56.764 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:56.821 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:56.935 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:57.039 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:57.097 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:57.211 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:57.311 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:57.368 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:57.478 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:57.579 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-17 zones={test-zone3, test-zone1} 2021-11-11T21:11:57.580 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-17 zones={test-zone3, test-zone2} 2021-11-11T21:11:57.588 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:57.593 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:57.593 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:57.593 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:57.594 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:57.594 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:57.599 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:57.600 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:57.600 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:57.601 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:57.655 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:57.670 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:57.734 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:57.792 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:57.848 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:57.959 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:58.059 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:58.117 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:58.224 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:11:58.332 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:58.393 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:58.505 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:11:58.603 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-17 zones={test-zone3, test-zone2} 2021-11-11T21:11:58.604 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:58.605 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:58.605 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:58.659 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:58.659 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:58.727 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:58.725+0000 7f4b04b1a380 1 bucket sync caught up with source: 2021-11-11T21:11:58.727 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000011.28.5, , 00000000007.43.5, , 00000000008.52.5, , , , , ] 2021-11-11T21:11:58.727 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000011.28.5, , 00000000007.43.5, , 00000000008.52.5, , , , , ] 2021-11-11T21:11:58.727 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:58.725+0000 7f4b04b1a380 0 bucket checkpoint complete 2021-11-11T21:11:58.740 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:58.741 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:58.742 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:58.792 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:58.792 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:58.864 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:58.864+0000 7f0fdc4e8380 1 bucket sync caught up with source: 2021-11-11T21:11:58.864 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000011.28.5, , 00000000007.43.5, , 00000000008.52.5, , , , , ] 2021-11-11T21:11:58.864 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000011.28.5, , 00000000007.43.5, , 00000000008.52.5, , , , , ] 2021-11-11T21:11:58.865 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:58.864+0000 7f0fdc4e8380 0 bucket checkpoint complete 2021-11-11T21:11:58.879 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:58.880 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:58.880 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:58.929 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:58.930 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:59.000 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:59.001+0000 7fc8ca32c380 1 bucket sync caught up with source: 2021-11-11T21:11:59.001 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000011.28.5, , 00000000007.43.5, , 00000000008.52.5, , , , , ] 2021-11-11T21:11:59.001 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000011.28.5, , 00000000007.43.5, , 00000000008.52.5, , , , , ] 2021-11-11T21:11:59.002 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:59.001+0000 7fc8ca32c380 0 bucket checkpoint complete 2021-11-11T21:11:59.016 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:59.017 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:59.017 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:59.067 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:59.068 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:59.125 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:59.126+0000 7fa59b796380 1 bucket sync caught up with source: 2021-11-11T21:11:59.126 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000004.68.5, , 00000000011.213.5, , 00000000008.259.5, , , , , ] 2021-11-11T21:11:59.126 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000004.68.5, , 00000000011.213.5, , 00000000008.259.5, , , , , ] 2021-11-11T21:11:59.126 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:59.126+0000 7fa59b796380 0 bucket checkpoint complete 2021-11-11T21:11:59.140 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:59.141 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:59.141 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:59.189 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:59.190 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:59.247 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:59.247+0000 7fb3060ae380 1 bucket sync caught up with source: 2021-11-11T21:11:59.247 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000004.68.5, , 00000000011.213.5, , 00000000008.259.5, , , , , ] 2021-11-11T21:11:59.247 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000004.68.5, , 00000000011.213.5, , 00000000008.259.5, , , , , ] 2021-11-11T21:11:59.248 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:59.247+0000 7fb3060ae380 0 bucket checkpoint complete 2021-11-11T21:11:59.262 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:59.263 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:59.263 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:59.311 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:59.312 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:59.382 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:59.382+0000 7f32dfd67380 1 bucket sync caught up with source: 2021-11-11T21:11:59.382 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, 00000000004.69.5, , 00000000007.228.5, , 00000000017.286.5, , , , , ] 2021-11-11T21:11:59.382 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, 00000000004.69.5, , 00000000007.228.5, , 00000000017.286.5, , , , , ] 2021-11-11T21:11:59.382 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:11:59.382+0000 7f32dfd67380 0 bucket checkpoint complete 2021-11-11T21:11:59.398 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:59.399 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-18', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:11:59.399 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-18 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:11:59.451 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:11:59.451 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:11:59.509 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:59.509+0000 7fdc7434a380 1 bucket sync caught up with source: 2021-11-11T21:11:59.509 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, 00000000004.69.5, , 00000000007.228.5, , 00000000017.286.5, , , , , ] 2021-11-11T21:11:59.509 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, 00000000004.69.5, , 00000000007.228.5, , 00000000017.286.5, , , , , ] 2021-11-11T21:11:59.510 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:11:59.509+0000 7fdc7434a380 0 bucket checkpoint complete 2021-11-11T21:11:59.524 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-18 zones={test-zone2, test-zone1} 2021-11-11T21:11:59.532 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:11:59.537 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:59.538 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:59.538 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:59.539 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:59.539 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:11:59.544 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:11:59.545 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:11:59.545 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:11:59.546 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:11:59.556 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:59.568 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:59.634 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:11:59.767 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:59.824 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:11:59.936 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:12:00.039 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:12:00.098 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:12:00.209 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:12:00.319 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:12:00.377 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:12:00.496 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:12:00.600 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-18 zones={test-zone2, test-zone1} 2021-11-11T21:12:00.600 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-18 zones={test-zone3, test-zone1} 2021-11-11T21:12:00.608 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:12:00.655 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:00.656 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:12:00.656 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:12:00.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:12:00.657 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:12:00.663 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:00.663 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:12:00.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:12:00.664 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:12:00.715 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:00.729 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:00.794 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:00.852 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:12:00.913 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:12:01.024 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:12:01.132 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:12:01.190 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:12:01.296 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:12:01.399 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:12:01.456 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:12:01.566 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:12:01.667 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-18 zones={test-zone3, test-zone1} 2021-11-11T21:12:01.668 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-18 zones={test-zone3, test-zone2} 2021-11-11T21:12:01.678 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:12:01.683 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:01.683 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:12:01.684 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:12:01.684 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:12:01.685 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:12:01.691 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:01.691 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone1 2021-11-11T21:12:01.692 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone2 2021-11-11T21:12:01.692 DEBUG:tasks.rgw_multi.zone_rados:o=obj-test-zone3 2021-11-11T21:12:01.702 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:01.716 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:01.779 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:01.836 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:12:01.897 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:12:02.007 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone1 2021-11-11T21:12:02.108 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:12:02.166 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:12:02.278 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone2 2021-11-11T21:12:02.381 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:12:02.438 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:12:02.558 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj-test-zone3 2021-11-11T21:12:02.660 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-18 zones={test-zone3, test-zone2} 2021-11-11T21:12:02.662 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_versioned_object_incremental_sync ... ok 2021-11-11T21:12:02.687 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:02.689 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:02.689 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:02.731 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:02.731 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:02.777 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:12:02.795 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:12:02.795 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:02.796 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:02.796 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:12:02.796 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:02.796 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:02.797 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:02.797 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:12:02.797 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:02.797 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:02.798 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665122.688130_117.1", 2021-11-11T21:12:02.798 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:02.688130Z" 2021-11-11T21:12:02.798 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:02.798 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:02.799 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:02.799 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:12:02.799 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:02.799 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:12:02.801 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:12:02.801 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:02.802 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:02.802 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:02.851 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:02.852 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:02.918 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:12:02.918 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:12:02.919 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:12:02.919 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:12:02.919 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:12:02.920 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:12:02.920 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:02.920 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:02.920 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:12:02.921 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:02.921 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:12:02.921 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:02.922 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:02.922 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:02.923 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:02.923 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:12:02.923 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:02.923 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:12:02.924 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:02.924 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:02.925 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:02.925 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:02.925 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:12:02.926 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:02.926 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:02.927 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:02.927 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:02.928 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:02.928 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:02.929 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:12:02.929 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:02.930 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:02.930 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:02.930 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:02.931 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:12:02.931 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:02.932 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:02.932 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665122.671807_112.1", 2021-11-11T21:12:02.932 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:02.933 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:02.933 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:02.933 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:02.671807Z", 2021-11-11T21:12:02.934 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:02.934 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:02.934 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:02.934 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:02.935 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:12:02.935 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:02.935 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:02.935 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:02.936 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:02.936 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:02.936 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:02.937 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:12:02.937 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:02.937 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:02.937 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:02.937 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:12:02.938 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:02.938 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:12:02.938 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:12:02.938 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:12:02.939 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:02.939 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:12:02.940 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665122.688130_117.1 target=1_1636665122.671807_112.1 2021-11-11T21:12:07.942 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:07.942 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:07.943 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:07.993 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:07.994 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:08.074 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:12:08.074 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:12:08.075 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:12:08.075 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:12:08.076 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:12:08.076 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:12:08.077 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:08.077 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:08.078 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:12:08.078 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:08.078 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:12:08.079 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:08.079 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:08.080 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:08.081 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:08.081 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:12:08.082 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:08.082 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:12:08.083 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:08.083 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.084 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:08.084 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:08.085 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:12:08.085 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:08.085 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:08.086 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:08.086 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:08.087 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:08.087 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:08.088 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:12:08.088 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:08.089 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.089 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:08.090 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:08.090 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:12:08.090 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:08.091 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:08.091 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665122.688130_117.1", 2021-11-11T21:12:08.092 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:08.092 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:08.093 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:08.093 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:02.688130Z", 2021-11-11T21:12:08.094 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:08.094 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.095 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:08.095 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:08.096 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:12:08.096 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:08.096 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:08.097 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:08.097 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:08.098 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:08.098 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:08.099 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:12:08.099 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:08.099 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.100 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.100 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:12:08.100 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:08.101 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:12:08.101 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:12:08.102 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:12:08.102 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.103 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:12:08.105 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:12:08.105 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:12:08.106 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:08.106 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:08.107 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:08.155 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:08.156 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:08.240 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:12:08.240 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:12:08.241 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:12:08.241 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:12:08.241 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:12:08.241 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:12:08.242 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:12:08.242 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:08.242 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:12:08.242 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:08.243 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:12:08.243 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:08.243 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:12:08.243 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:08.244 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:08.244 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:12:08.244 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:08.244 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:12:08.245 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:12:08.245 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:08.245 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:08.245 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:08.245 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:12:08.246 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:08.246 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:12:08.246 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:08.246 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:08.247 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:08.247 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:08.247 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:12:08.247 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:12:08.248 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:08.248 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:08.248 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:08.248 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:12:08.249 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:08.249 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:12:08.249 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665122.688130_117.1", 2021-11-11T21:12:08.249 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:08.249 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:08.250 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:08.250 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:02.688130Z", 2021-11-11T21:12:08.250 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:12:08.250 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:08.251 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:08.251 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:08.251 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:12:08.251 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:08.252 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:12:08.252 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:08.252 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:08.252 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:08.252 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:08.253 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:12:08.253 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:12:08.253 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:08.253 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:08.254 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:12:08.254 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:08.254 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:12:08.254 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:12:08.254 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:12:08.255 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:08.255 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:12:08.256 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:12:08.257 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:12:08.257 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:08.258 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:08.258 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:08.300 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:08.301 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:08.373 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:12:08.374 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:12:08.374 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:12:08.374 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:12:08.374 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:12:08.375 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:12:08.375 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:08.375 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:08.375 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:12:08.376 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:08.376 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:12:08.376 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:08.377 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:08.377 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:08.377 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:08.377 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:12:08.378 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:08.378 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:12:08.378 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:08.378 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.379 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:08.379 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:08.379 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:12:08.379 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:08.380 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:08.380 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:08.380 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:08.380 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:08.380 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:08.381 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:12:08.381 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:08.381 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.382 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:08.382 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:08.382 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:12:08.383 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:08.383 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:08.383 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665122.688130_117.1", 2021-11-11T21:12:08.383 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:08.384 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:08.384 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:08.384 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:02.688130Z", 2021-11-11T21:12:08.384 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:08.385 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.385 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:08.385 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:08.385 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:12:08.386 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:08.386 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:08.386 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:08.386 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:08.386 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:08.387 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:08.387 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:12:08.387 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:08.387 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.388 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.388 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:12:08.388 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:08.388 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:12:08.389 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:12:08.389 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:12:08.389 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:08.389 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:12:08.391 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:12:08.439 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:08.440 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:08.440 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:08.491 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:08.491 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:08.554 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:08.555+0000 7fc36aee7380 1 bucket sync caught up with source: 2021-11-11T21:12:08.555 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , 00000000002.14.10, , , , , ] 2021-11-11T21:12:08.555 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000002.14.10, , , , , ] 2021-11-11T21:12:08.556 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:08.555+0000 7fc36aee7380 0 bucket checkpoint complete 2021-11-11T21:12:08.569 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:08.571 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:08.572 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:08.624 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:08.624 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:08.692 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:08.693+0000 7faca40d6380 1 bucket sync caught up with source: 2021-11-11T21:12:08.693 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000002.14.10, , , , , ] 2021-11-11T21:12:08.693 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000002.14.10, , , , , ] 2021-11-11T21:12:08.693 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:08.693+0000 7faca40d6380 0 bucket checkpoint complete 2021-11-11T21:12:08.715 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:08.716 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:08.717 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:08.768 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:08.768 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:08.831 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:08.832+0000 7fd451d1b380 1 waiting for incremental sync to catch up: 2021-11-11T21:12:08.832 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , 00000000002.14.10, , , , , ] 2021-11-11T21:12:08.832 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000004.18.9, , , , , ] 2021-11-11T21:12:13.833 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:13.833+0000 7fd451d1b380 1 bucket sync caught up with source: 2021-11-11T21:12:13.833 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , 00000000006.21.9, , , , , ] 2021-11-11T21:12:13.834 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000004.18.9, , , , , ] 2021-11-11T21:12:13.834 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:13.833+0000 7fd451d1b380 0 bucket checkpoint complete 2021-11-11T21:12:13.850 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:13.851 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:13.852 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:13.906 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:13.907 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:13.974 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:13.974+0000 7f4246fe8380 1 bucket sync caught up with source: 2021-11-11T21:12:13.974 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000004.271.9, , , , , ] 2021-11-11T21:12:13.974 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000004.271.9, , , , , ] 2021-11-11T21:12:13.975 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:13.974+0000 7f4246fe8380 0 bucket checkpoint complete 2021-11-11T21:12:13.998 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:13.999 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:13.999 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:14.048 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:14.048 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:14.120 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:14.120+0000 7f650fd9a380 1 bucket sync caught up with source: 2021-11-11T21:12:14.120 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000004.271.9, , , , , ] 2021-11-11T21:12:14.120 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000004.271.9, , , , , ] 2021-11-11T21:12:14.120 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:14.120+0000 7f650fd9a380 0 bucket checkpoint complete 2021-11-11T21:12:14.139 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:14.140 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:14.140 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:14.189 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:14.190 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:14.256 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:14.256+0000 7f8b096dc380 1 bucket sync caught up with source: 2021-11-11T21:12:14.256 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000004.270.9, , , , , ] 2021-11-11T21:12:14.256 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000004.270.9, , , , , ] 2021-11-11T21:12:14.256 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:14.256+0000 7f8b096dc380 0 bucket checkpoint complete 2021-11-11T21:12:14.271 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:14.272 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:14.273 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:14.330 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:14.331 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:14.391 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:14.392+0000 7f3158ba4380 1 bucket sync caught up with source: 2021-11-11T21:12:14.392 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , 00000000004.270.9, , , , , ] 2021-11-11T21:12:14.392 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000004.270.9, , , , , ] 2021-11-11T21:12:14.392 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:14.392+0000 7f3158ba4380 0 bucket checkpoint complete 2021-11-11T21:12:14.407 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-19 zones={test-zone2, test-zone1} 2021-11-11T21:12:14.416 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:12:14.463 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:14.464 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:12:14.468 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:14.519 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:14.533 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:14.599 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:14.655 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-19 zones={test-zone2, test-zone1} 2021-11-11T21:12:14.656 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-19 zones={test-zone3, test-zone1} 2021-11-11T21:12:14.664 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:12:14.716 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:14.717 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:12:14.722 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:14.772 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:14.785 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:14.855 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:14.911 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-19 zones={test-zone3, test-zone1} 2021-11-11T21:12:14.912 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-19 zones={test-zone3, test-zone2} 2021-11-11T21:12:14.920 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:12:14.971 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:14.972 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:12:14.976 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:15.028 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:15.041 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:15.107 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:15.163 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-19 zones={test-zone3, test-zone2} 2021-11-11T21:12:16.695 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:16.696 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:16.696 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:16.748 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:16.749 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:16.808 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:16.808+0000 7f958b441380 1 waiting for incremental sync to catch up: 2021-11-11T21:12:16.808 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , 00000000006.21.9, , , 00000000047.121.11, , ] 2021-11-11T21:12:16.808 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000006.21.9, , , 00000000052.134.11, , ] 2021-11-11T21:12:21.809 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:21.810+0000 7f958b441380 1 bucket sync caught up with source: 2021-11-11T21:12:21.810 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , 00000000006.21.9, , , 00000000053.136.9, , ] 2021-11-11T21:12:21.810 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000006.21.9, , , 00000000052.134.11, , ] 2021-11-11T21:12:21.810 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:21.810+0000 7f958b441380 0 bucket checkpoint complete 2021-11-11T21:12:21.829 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:21.830 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:21.830 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:21.885 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:21.885 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:21.952 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:21.953+0000 7f8bb1f0f380 1 bucket sync caught up with source: 2021-11-11T21:12:21.952 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000006.21.9, , , 00000000053.136.9, , ] 2021-11-11T21:12:21.953 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000006.21.9, , , 00000000053.136.9, , ] 2021-11-11T21:12:21.953 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:21.953+0000 7f8bb1f0f380 0 bucket checkpoint complete 2021-11-11T21:12:21.968 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:21.968 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:21.969 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:22.023 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:22.023 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:22.093 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:22.094+0000 7f1a72fd8380 1 bucket sync caught up with source: 2021-11-11T21:12:22.094 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , 00000000006.21.9, , , 00000000053.136.9, , ] 2021-11-11T21:12:22.094 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000006.21.9, , , 00000000053.136.9, , ] 2021-11-11T21:12:22.094 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:22.094+0000 7f1a72fd8380 0 bucket checkpoint complete 2021-11-11T21:12:22.109 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:22.110 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:22.111 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:22.162 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:22.163 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:22.230 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:22.230+0000 7fa72b30a380 1 bucket sync caught up with source: 2021-11-11T21:12:22.230 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000004.271.9, , , 00000000060.301.9, , ] 2021-11-11T21:12:22.230 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000004.271.9, , , 00000000060.301.9, , ] 2021-11-11T21:12:22.231 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:22.230+0000 7fa72b30a380 0 bucket checkpoint complete 2021-11-11T21:12:22.247 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:22.248 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:22.248 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:22.297 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:22.298 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:22.362 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:22.363+0000 7fe59e937380 1 bucket sync caught up with source: 2021-11-11T21:12:22.363 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000004.271.9, , , 00000000060.301.9, , ] 2021-11-11T21:12:22.363 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000004.271.9, , , 00000000060.301.9, , ] 2021-11-11T21:12:22.363 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:22.363+0000 7fe59e937380 0 bucket checkpoint complete 2021-11-11T21:12:22.378 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:22.379 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:22.379 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:22.428 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:22.429 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:22.489 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:22.489+0000 7f74b4570380 1 bucket sync caught up with source: 2021-11-11T21:12:22.489 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , 00000000004.270.9, , , 00000000065.310.5, , ] 2021-11-11T21:12:22.489 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , 00000000004.270.9, , , 00000000065.310.5, , ] 2021-11-11T21:12:22.490 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:22.489+0000 7f74b4570380 0 bucket checkpoint complete 2021-11-11T21:12:22.506 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:22.507 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-19', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:22.507 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-19 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:22.557 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:22.557 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:22.614 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:22.615+0000 7f06d3854380 1 bucket sync caught up with source: 2021-11-11T21:12:22.615 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , 00000000004.270.9, , , 00000000065.310.5, , ] 2021-11-11T21:12:22.615 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , 00000000004.270.9, , , 00000000065.310.5, , ] 2021-11-11T21:12:22.615 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:22.615+0000 7f06d3854380 0 bucket checkpoint complete 2021-11-11T21:12:22.629 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-19 zones={test-zone2, test-zone1} 2021-11-11T21:12:22.638 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:12:22.645 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:22.645 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.646 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.646 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.647 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.647 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.648 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.648 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.649 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.649 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.650 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.650 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.651 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.651 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.651 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.652 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.652 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.653 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.653 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.654 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.654 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.655 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.655 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.656 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.656 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.657 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.658 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.658 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.659 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.659 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.660 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:12:22.667 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:22.667 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.668 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.668 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.669 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.669 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.670 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.671 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.671 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.672 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.672 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.673 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.673 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.673 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.674 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.674 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.675 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.675 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.676 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.676 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.676 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.677 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.677 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.678 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.678 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.679 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.679 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.680 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.680 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:22.693 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:22.705 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:22.771 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:22.829 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:22.886 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:22.998 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:23.104 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:23.160 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:23.269 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:23.372 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:23.429 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:23.542 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:23.648 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:23.706 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:23.820 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:23.924 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:23.981 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:24.143 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:24.248 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:24.310 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:24.427 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:24.537 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:24.594 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:24.711 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:24.821 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:24.879 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:24.999 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:25.112 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:25.174 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:25.291 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:25.396 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:25.456 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:25.572 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:25.680 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:25.739 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:25.859 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:25.964 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:26.026 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:26.144 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:26.252 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:26.310 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:26.424 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:26.537 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:26.594 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:26.716 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:26.825 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:26.882 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:26.998 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:27.112 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:27.174 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:27.287 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:27.392 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:27.451 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:27.567 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:27.677 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:27.734 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:27.848 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:27.952 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:28.013 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:28.128 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:28.236 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:28.298 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:28.419 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:28.529 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:28.586 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:28.703 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:28.813 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:28.871 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:28.992 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:29.100 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:29.157 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:29.270 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:29.376 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:29.438 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:29.556 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:29.660 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:29.722 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:29.838 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:29.940 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:29.998 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:30.116 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:30.224 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:30.282 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:30.395 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:30.500 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:30.559 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:30.676 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:30.784 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:30.843 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:30.959 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:31.064 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:31.122 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:31.241 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:31.344 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-19 zones={test-zone2, test-zone1} 2021-11-11T21:12:31.345 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-19 zones={test-zone3, test-zone1} 2021-11-11T21:12:31.355 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:12:31.362 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:31.362 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.363 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.364 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.364 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.364 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.365 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.365 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.366 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.366 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.367 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.367 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.368 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.368 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.368 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.369 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.369 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.370 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.370 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.371 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.371 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.371 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.372 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.372 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.373 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.373 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.374 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.374 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.375 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.375 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.375 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.376 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:12:31.384 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:31.384 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.385 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.385 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.386 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.386 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.386 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.387 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.387 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.388 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.388 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.389 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.389 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.389 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.390 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.390 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.391 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.391 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.392 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.392 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.393 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.393 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.393 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.394 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.394 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.395 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.395 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.396 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.396 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.397 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.397 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:31.411 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:31.426 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:31.496 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:31.560 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:31.617 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:31.733 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:31.839 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:31.896 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:32.009 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:32.116 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:32.173 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:32.285 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:32.395 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:32.453 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:32.567 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:32.672 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:32.729 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:32.839 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:32.944 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:33.000 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:33.113 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:33.212 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:33.273 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:33.382 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:33.492 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:33.549 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:33.658 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:33.759 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:33.817 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:33.925 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:34.023 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:34.081 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:34.196 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:34.308 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:34.401 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:34.539 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:34.643 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:34.701 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:34.812 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:34.911 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:34.970 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:35.081 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:35.189 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:35.248 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:35.365 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:35.464 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:35.521 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:35.638 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:35.739 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:35.796 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:35.904 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.003 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.060 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.176 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.280 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.337 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.444 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.543 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.600 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.709 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.816 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.874 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:36.989 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:37.091 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:37.153 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:37.265 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:37.367 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:37.428 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:37.541 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:37.649 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:37.704 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:37.812 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:37.919 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:37.977 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:38.093 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:38.195 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:38.256 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:38.364 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:38.464 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:38.520 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:38.636 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:38.740 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:38.797 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:38.909 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:39.007 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:39.064 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:39.178 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:39.280 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:39.341 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:39.449 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:39.555 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:39.616 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:39.725 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:39.824 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-19 zones={test-zone3, test-zone1} 2021-11-11T21:12:39.824 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-19 zones={test-zone3, test-zone2} 2021-11-11T21:12:39.834 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:12:39.840 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:39.841 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.841 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.842 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.842 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.843 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.843 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.844 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.844 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.845 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.845 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.846 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.847 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.848 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.848 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.849 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.849 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.849 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.850 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.850 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.852 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.852 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.853 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.854 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.854 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.856 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.856 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.857 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.857 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:12:39.866 DEBUG:tasks.rgw_multi.zone_rados:o=dummy 2021-11-11T21:12:39.866 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.867 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.867 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.868 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.868 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.868 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.869 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.869 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.870 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.870 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.871 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.871 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.871 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.872 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.873 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.873 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.873 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.874 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.875 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.875 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.876 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.876 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.877 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.877 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.877 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.878 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.878 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.879 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.879 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.880 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:39.894 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:39.909 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:39.975 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=dummy 2021-11-11T21:12:40.032 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:40.092 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:40.211 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:40.316 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:40.373 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:40.490 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:40.594 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:40.650 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:40.762 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:40.869 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:40.925 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:41.039 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:41.144 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:41.205 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:41.325 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:41.436 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:41.494 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:41.607 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:41.712 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:41.772 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:41.887 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:41.996 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:42.059 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:42.180 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:42.285 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:42.346 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:42.467 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:42.576 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:42.641 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:42.760 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:42.864 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:42.926 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:43.043 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:43.144 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:43.211 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:43.335 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:43.440 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:43.498 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:43.612 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:43.720 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:43.778 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:43.895 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:44.010 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:44.070 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:44.196 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:44.308 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:44.366 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:44.480 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:44.584 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:44.643 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:44.769 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:44.881 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:44.938 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:45.056 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:45.160 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:45.222 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:45.336 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:45.448 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:45.511 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:45.634 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:45.736 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:45.797 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:45.914 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:46.020 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:46.082 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:46.208 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:46.316 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:46.374 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:46.491 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:46.596 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:46.671 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:46.792 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:46.904 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:46.966 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:47.088 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:47.196 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:47.259 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:47.374 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:47.477 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:47.534 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:47.648 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:47.757 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:47.818 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:47.935 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:48.049 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:48.107 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:48.223 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:48.330 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:48.386 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:48.504 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:48.612 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-19 zones={test-zone3, test-zone2} 2021-11-11T21:12:48.614 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:48.615 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:48.616 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:48.666 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:48.666 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:48.739 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:12:48.739 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:48.740 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:48.740 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:07.135176Z" 2021-11-11T21:12:48.740 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:48.740 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:48.741 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:48.741 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:29.139957Z" 2021-11-11T21:12:48.741 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:48.742 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:48.742 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:48.742 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:29.141661Z" 2021-11-11T21:12:48.743 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:48.743 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:48.743 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:48.743 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:12:48.744 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:48.745 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:12:48.746 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone2 source_zone=test-zone1 2021-11-11T21:12:48.746 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:48.747 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:48.748 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:48.799 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:48.800 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:48.878 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:12:48.878 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:12:48.878 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:12:48.879 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:12:48.879 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:12:48.879 INFO:teuthology.orchestra.run.smithi198.stdout: "instance_id": 16065388824966449568 2021-11-11T21:12:48.879 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:48.880 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:12:48.880 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:48.880 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:12:48.880 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:48.880 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:48.881 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:48.881 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:48.881 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:48.881 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:48.882 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:07.135176Z" 2021-11-11T21:12:48.882 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:48.882 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:48.882 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:48.883 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:12:48.883 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:48.883 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:48.883 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:48.883 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:48.884 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:48.884 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:48.884 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:29.139957Z" 2021-11-11T21:12:48.884 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:48.885 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:48.885 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:48.885 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:12:48.885 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:48.886 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:48.886 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:48.886 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:48.886 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:48.886 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:48.887 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:29.141661Z" 2021-11-11T21:12:48.887 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:48.887 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:48.888 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:48.888 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:12:48.888 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:48.888 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:48.888 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:48.889 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:48.889 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:48.889 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:48.889 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000" 2021-11-11T21:12:48.890 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:48.890 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:48.890 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:12:48.890 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:48.890 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:12:48.891 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 0, 2021-11-11T21:12:48.891 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 0 2021-11-11T21:12:48.891 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:48.891 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:12:48.893 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone2 source_zone=test-zone1 2021-11-11T21:12:48.893 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:48.894 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:48.894 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:48.936 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:48.937 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:49.026 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:12:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.027 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:07.135176Z" 2021-11-11T21:12:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.028 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:29.139957Z" 2021-11-11T21:12:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.029 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:29.141661Z" 2021-11-11T21:12:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.030 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:12:49.031 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.031 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:12:49.033 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone3 source_zone=test-zone1 2021-11-11T21:12:49.033 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.034 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.035 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:49.117 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:49.117 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:49.198 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:12:49.199 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:12:49.199 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:12:49.199 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:12:49.200 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:12:49.200 INFO:teuthology.orchestra.run.smithi096.stdout: "instance_id": 4372551904339153811 2021-11-11T21:12:49.200 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.200 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:12:49.201 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.201 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:12:49.201 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:49.201 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.201 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:49.202 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:49.202 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:49.202 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:49.202 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:07.135176Z" 2021-11-11T21:12:49.203 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.203 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.203 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.203 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:12:49.204 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:49.204 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.204 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:49.204 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:49.205 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:49.205 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:49.205 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:29.139957Z" 2021-11-11T21:12:49.205 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.206 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.206 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.206 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:12:49.206 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:49.207 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.207 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:49.207 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:49.207 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:49.208 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:49.208 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:29.141661Z" 2021-11-11T21:12:49.208 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.208 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.209 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.209 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:12:49.209 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:49.209 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.210 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:49.210 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:49.210 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:49.210 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:49.211 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000" 2021-11-11T21:12:49.211 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.211 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.211 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:12:49.212 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.212 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:12:49.212 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 0, 2021-11-11T21:12:49.212 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 0 2021-11-11T21:12:49.213 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.213 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:12:49.214 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone3 source_zone=test-zone1 2021-11-11T21:12:49.215 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.215 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.216 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:49.295 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:49.295 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:49.364 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:12:49.365 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.365 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:49.365 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:07.135176Z" 2021-11-11T21:12:49.365 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.366 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.366 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:49.366 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:29.139957Z" 2021-11-11T21:12:49.366 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.366 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.367 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:49.367 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:29.141661Z" 2021-11-11T21:12:49.367 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.367 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.368 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:49.368 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:12:49.368 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.368 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:12:49.371 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone4 source_zone=test-zone1 2021-11-11T21:12:49.371 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.373 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone1', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.373 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone1 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:49.419 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:49.420 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:49.498 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:12:49.498 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:12:49.499 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:12:49.499 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:12:49.499 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:12:49.499 INFO:teuthology.orchestra.run.smithi198.stdout: "instance_id": 10067931636928000873 2021-11-11T21:12:49.500 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:49.500 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:12:49.500 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.500 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:12:49.501 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:49.501 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.501 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:49.502 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:49.502 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:49.502 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:49.503 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:07.135176Z" 2021-11-11T21:12:49.503 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:49.503 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:49.503 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.503 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:12:49.504 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:49.504 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.504 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:49.504 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:49.505 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:49.505 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:49.505 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:29.139957Z" 2021-11-11T21:12:49.505 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:49.505 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:49.506 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.506 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:12:49.506 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:49.506 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.507 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:49.507 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:49.507 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:49.507 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:49.507 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:29.141661Z" 2021-11-11T21:12:49.508 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:49.508 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:49.508 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.508 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:12:49.509 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:49.509 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.509 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:49.509 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:49.510 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:49.510 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:49.510 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000" 2021-11-11T21:12:49.510 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:49.511 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:49.511 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:12:49.511 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:49.511 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:12:49.512 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 0, 2021-11-11T21:12:49.512 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 0 2021-11-11T21:12:49.512 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:49.512 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:12:49.514 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone4 source_zone=test-zone1 2021-11-11T21:12:49.514 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.515 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.515 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:49.557 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:49.558 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:49.632 INFO:teuthology.orchestra.run.smithi198.stdout:[ 2021-11-11T21:12:49.633 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.633 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:49.633 INFO:teuthology.orchestra.run.smithi198.stdout: "last_update": "2021-11-11T21:12:07.053106Z" 2021-11-11T21:12:49.634 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:49.634 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.634 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:49.634 INFO:teuthology.orchestra.run.smithi198.stdout: "last_update": "2021-11-11T21:12:29.057113Z" 2021-11-11T21:12:49.635 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:49.635 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.635 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:49.635 INFO:teuthology.orchestra.run.smithi198.stdout: "last_update": "2021-11-11T21:12:29.058735Z" 2021-11-11T21:12:49.636 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:49.636 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.636 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:49.636 INFO:teuthology.orchestra.run.smithi198.stdout: "last_update": "0.000000" 2021-11-11T21:12:49.636 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:49.637 INFO:teuthology.orchestra.run.smithi198.stdout:] 2021-11-11T21:12:49.638 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone1 source_zone=test-zone2 2021-11-11T21:12:49.638 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.639 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.640 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:49.684 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:49.685 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:49.768 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:12:49.769 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:12:49.769 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:12:49.769 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:12:49.770 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:12:49.770 INFO:teuthology.orchestra.run.smithi096.stdout: "instance_id": 12438804399084526938 2021-11-11T21:12:49.770 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.771 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:12:49.771 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.771 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:12:49.772 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:49.772 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.772 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:49.773 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:49.773 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:49.773 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:49.774 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:07.053106Z" 2021-11-11T21:12:49.774 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.775 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.775 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.775 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:12:49.776 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:49.776 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.776 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:49.777 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:49.777 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:49.777 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:49.778 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:29.057113Z" 2021-11-11T21:12:49.778 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.779 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.779 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.779 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:12:49.780 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:49.780 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.781 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:49.781 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:49.781 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:49.782 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:49.782 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:29.058735Z" 2021-11-11T21:12:49.782 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.783 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.783 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:49.784 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:12:49.784 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:49.784 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:49.784 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:49.785 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:49.785 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:49.785 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:49.785 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000" 2021-11-11T21:12:49.785 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.786 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.786 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:12:49.786 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:49.787 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:12:49.787 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 0, 2021-11-11T21:12:49.787 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 0 2021-11-11T21:12:49.788 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:49.788 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:12:49.790 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone1 source_zone=test-zone2 2021-11-11T21:12:49.791 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.792 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'datalog', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.793 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 datalog status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:49.835 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:49.835 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:49.909 INFO:teuthology.orchestra.run.smithi198.stdout:[ 2021-11-11T21:12:49.910 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.910 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:49.910 INFO:teuthology.orchestra.run.smithi198.stdout: "last_update": "2021-11-11T21:12:07.053106Z" 2021-11-11T21:12:49.911 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:49.911 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.911 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:49.911 INFO:teuthology.orchestra.run.smithi198.stdout: "last_update": "2021-11-11T21:12:29.057113Z" 2021-11-11T21:12:49.912 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:49.912 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.912 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:49.912 INFO:teuthology.orchestra.run.smithi198.stdout: "last_update": "2021-11-11T21:12:29.058735Z" 2021-11-11T21:12:49.912 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:49.913 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:49.913 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:49.913 INFO:teuthology.orchestra.run.smithi198.stdout: "last_update": "0.000000" 2021-11-11T21:12:49.913 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:49.914 INFO:teuthology.orchestra.run.smithi198.stdout:] 2021-11-11T21:12:49.915 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone3 source_zone=test-zone2 2021-11-11T21:12:49.915 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.916 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone2', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:49.916 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone2 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:49.961 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:49.961 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:50.035 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:12:50.036 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:12:50.036 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:12:50.036 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:12:50.037 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:12:50.037 INFO:teuthology.orchestra.run.smithi096.stdout: "instance_id": 12905050393586868315 2021-11-11T21:12:50.037 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.037 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:12:50.038 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.038 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:12:50.039 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:50.039 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.039 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:50.040 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:50.040 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:50.040 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:50.040 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:07.053106Z" 2021-11-11T21:12:50.041 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.041 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.041 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.041 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:12:50.042 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:50.042 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.042 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:50.042 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:50.043 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:50.043 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:50.043 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:29.057113Z" 2021-11-11T21:12:50.043 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.044 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.044 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.044 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:12:50.044 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:50.045 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.045 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:50.046 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:50.046 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:50.046 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:50.046 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:29.058735Z" 2021-11-11T21:12:50.047 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.047 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.047 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.048 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:12:50.048 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:50.048 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.048 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:50.049 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:50.049 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:50.049 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:50.049 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000" 2021-11-11T21:12:50.050 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.050 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.050 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:12:50.050 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.050 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:12:50.051 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 0, 2021-11-11T21:12:50.051 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 0 2021-11-11T21:12:50.051 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.051 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:12:50.053 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone3 source_zone=test-zone2 2021-11-11T21:12:50.053 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.054 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.054 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:50.134 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:50.135 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:50.212 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:12:50.213 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.213 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:50.213 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:07.037039Z" 2021-11-11T21:12:50.213 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.214 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.214 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:50.214 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:29.041182Z" 2021-11-11T21:12:50.214 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.215 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.215 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:50.215 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:29.042952Z" 2021-11-11T21:12:50.215 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.216 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.216 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:50.216 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:12:50.216 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.216 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:12:50.218 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone1 source_zone=test-zone3 2021-11-11T21:12:50.218 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.219 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'data', 'sync', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.220 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 data sync status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:50.302 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:50.303 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:50.370 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:12:50.371 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:12:50.371 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:12:50.371 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:12:50.372 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:12:50.372 INFO:teuthology.orchestra.run.smithi096.stdout: "instance_id": 10140799890006012796 2021-11-11T21:12:50.372 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.372 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:12:50.373 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.373 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:12:50.373 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:50.373 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.373 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:50.374 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:50.374 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:50.374 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:50.374 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:07.037039Z" 2021-11-11T21:12:50.375 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.375 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.375 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.375 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:12:50.376 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:50.376 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.376 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:50.377 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:50.377 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:50.377 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:50.377 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:29.041182Z" 2021-11-11T21:12:50.378 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.378 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.378 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.378 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:12:50.379 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:50.379 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.379 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:50.379 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:50.380 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:50.380 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:50.380 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:29.042952Z" 2021-11-11T21:12:50.381 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.381 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.381 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.381 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:12:50.381 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:50.382 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.382 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:50.382 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:50.383 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:50.383 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:50.384 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000" 2021-11-11T21:12:50.385 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.385 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.385 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:12:50.385 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.386 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:12:50.386 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 0, 2021-11-11T21:12:50.386 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 0 2021-11-11T21:12:50.386 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.387 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:12:50.388 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone1 source_zone=test-zone3 2021-11-11T21:12:50.388 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.389 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'datalog', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.390 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 datalog status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:50.469 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:50.470 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:50.541 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:12:50.541 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.542 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:50.542 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:07.037039Z" 2021-11-11T21:12:50.542 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.542 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.543 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:50.543 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:29.041182Z" 2021-11-11T21:12:50.543 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.543 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.544 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:50.544 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:29.042952Z" 2021-11-11T21:12:50.544 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.544 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.544 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:50.545 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:12:50.545 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.545 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:12:50.546 INFO:rgw_multi.tests:starting data checkpoint for target_zone=test-zone2 source_zone=test-zone3 2021-11-11T21:12:50.547 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.547 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'data', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--source-zone', 'test-zone3', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.548 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 data sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --source-zone test-zone3 --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:50.594 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:50.595 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:50.676 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:12:50.676 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:12:50.677 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:12:50.677 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:12:50.677 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:12:50.677 INFO:teuthology.orchestra.run.smithi198.stdout: "instance_id": 17038693961297880578 2021-11-11T21:12:50.677 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:50.678 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:12:50.678 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:50.678 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:12:50.678 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:50.679 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.679 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000002374", 2021-11-11T21:12:50.679 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:50.679 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:50.680 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:50.680 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:07.037039Z" 2021-11-11T21:12:50.680 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.680 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:50.681 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:50.681 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:12:50.681 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:50.681 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.682 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000001044", 2021-11-11T21:12:50.682 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:50.682 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:50.683 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:50.684 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:29.041182Z" 2021-11-11T21:12:50.684 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.684 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:50.684 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:50.685 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:12:50.685 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:50.685 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.685 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "00000000000000000000:00000000000000004236", 2021-11-11T21:12:50.685 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:50.686 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:50.686 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:50.686 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:29.042952Z" 2021-11-11T21:12:50.686 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.687 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:50.687 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:50.687 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:12:50.687 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:50.688 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "incremental-sync", 2021-11-11T21:12:50.688 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:50.688 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:50.688 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:50.689 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:50.689 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000" 2021-11-11T21:12:50.689 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.689 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.690 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:12:50.690 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:50.690 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:12:50.690 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 0, 2021-11-11T21:12:50.691 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 0 2021-11-11T21:12:50.691 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.691 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:12:50.692 INFO:rgw_multi.tests:finished data checkpoint for target_zone=test-zone2 source_zone=test-zone3 2021-11-11T21:12:50.692 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_concurrent_versioned_object_incremental_sync ... ok 2021-11-11T21:12:50.721 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.722 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.722 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:50.763 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:50.764 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:50.822 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:12:50.845 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:12:50.846 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.847 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:50.847 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:12:50.847 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.848 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.848 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:50.849 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:12:50.849 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.850 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.850 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665170.719213_121.1", 2021-11-11T21:12:50.850 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:50.719213Z" 2021-11-11T21:12:50.851 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:50.851 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:50.852 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:50.852 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:12:50.852 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:50.853 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:12:50.854 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:12:50.855 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.856 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:50.857 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:50.900 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:50.901 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:50.965 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:12:50.966 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:12:50.966 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:12:50.967 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:12:50.967 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:12:50.967 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:12:50.968 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:50.968 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:50.969 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:12:50.969 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:50.969 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:12:50.970 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:50.970 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:50.971 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:50.971 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:50.972 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:12:50.972 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:50.972 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:12:50.973 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:50.973 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.974 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:50.974 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:50.974 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:12:50.975 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:50.975 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:50.975 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:50.976 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:50.976 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:50.976 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:50.977 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:12:50.977 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:50.978 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.978 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:50.978 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:50.979 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:12:50.979 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:50.980 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:50.980 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665170.706307_118.1", 2021-11-11T21:12:50.981 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:50.981 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:50.981 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:50.982 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:50.706307Z", 2021-11-11T21:12:50.982 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:50.983 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.983 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:50.984 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:50.984 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:12:50.984 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:50.985 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:50.985 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:50.985 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:50.985 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:50.986 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:50.986 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:12:50.986 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:50.986 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.987 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.987 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:12:50.987 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:50.987 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:12:50.988 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:12:50.988 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:12:50.988 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:50.988 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:12:50.989 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665170.719213_121.1 target=1_1636665170.706307_118.1 2021-11-11T21:12:55.991 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:55.992 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:55.992 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:56.041 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:56.042 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:56.109 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:12:56.109 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:12:56.109 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:12:56.110 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:12:56.110 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:12:56.110 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:12:56.110 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:56.111 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:56.111 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:12:56.111 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:56.112 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:12:56.112 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:56.112 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:56.112 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:56.112 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:56.113 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:12:56.113 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:56.113 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:12:56.113 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:56.114 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.114 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:56.114 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:56.115 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:12:56.115 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:56.115 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:56.115 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:56.115 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:56.116 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:56.116 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:56.116 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:12:56.116 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:56.117 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.117 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:56.117 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:56.117 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:12:56.118 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:56.118 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:56.118 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665170.719213_121.1", 2021-11-11T21:12:56.118 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:56.119 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:56.119 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:56.119 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:50.719213Z", 2021-11-11T21:12:56.119 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:56.119 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.120 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:56.120 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:56.120 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:12:56.121 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:56.121 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:56.121 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:56.121 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:56.121 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:56.122 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:56.122 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:12:56.122 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:56.122 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.123 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.123 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:12:56.123 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:56.123 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:12:56.124 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:12:56.124 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:12:56.125 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.125 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:12:56.126 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:12:56.127 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:12:56.127 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:56.128 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:56.128 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:56.177 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:56.177 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:56.263 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:12:56.263 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:12:56.264 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:12:56.264 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:12:56.264 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:12:56.264 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:12:56.265 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:12:56.265 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:56.265 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:12:56.265 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:56.266 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:12:56.266 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:56.267 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:12:56.267 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:56.267 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:56.267 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:12:56.268 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:56.268 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:12:56.268 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:12:56.269 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:56.269 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:56.269 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:56.269 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:12:56.270 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:56.270 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:12:56.270 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:56.270 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:56.270 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:56.271 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:56.271 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:12:56.271 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:12:56.271 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:56.272 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:56.272 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:56.272 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:12:56.272 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:56.273 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:12:56.273 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665170.719213_121.1", 2021-11-11T21:12:56.273 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:56.273 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:56.273 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:56.274 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:50.719213Z", 2021-11-11T21:12:56.274 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:12:56.274 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:56.274 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:56.275 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:56.275 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:12:56.275 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:12:56.275 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:12:56.276 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:56.276 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:12:56.276 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:12:56.276 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:12:56.277 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:12:56.277 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:12:56.277 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:56.277 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:56.278 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:12:56.278 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:56.278 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:12:56.278 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:12:56.279 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:12:56.279 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:56.279 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:12:56.280 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:12:56.281 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:12:56.281 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:56.282 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:56.283 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:56.327 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:56.327 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:56.391 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:12:56.391 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:12:56.392 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:12:56.392 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:12:56.392 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:12:56.392 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:12:56.393 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:56.393 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:56.393 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:12:56.393 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:56.393 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:12:56.394 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:56.394 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:56.394 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:56.394 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:56.395 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:12:56.395 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:56.395 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:12:56.395 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:56.396 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.397 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:56.397 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:56.397 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:12:56.397 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:56.398 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:56.398 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:56.398 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:56.399 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:56.399 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:56.399 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:12:56.399 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:56.400 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.400 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:56.400 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:56.400 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:12:56.400 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:56.401 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:56.401 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665170.719213_121.1", 2021-11-11T21:12:56.401 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:56.401 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:56.402 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:56.402 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:50.719213Z", 2021-11-11T21:12:56.402 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:56.402 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.402 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:56.403 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:56.403 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:12:56.403 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:56.403 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:56.404 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:56.404 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:56.404 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:56.404 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:56.404 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:12:56.405 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:56.405 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.405 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.405 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:12:56.406 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:56.406 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:12:56.406 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:12:56.406 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:12:56.407 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:56.407 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:12:56.408 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:12:56.459 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:56.460 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:56.461 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:56.512 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:56.513 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:56.579 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:56.579+0000 7fe72a0ed380 1 bucket sync caught up with source: 2021-11-11T21:12:56.579 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000001.11.5, , ] 2021-11-11T21:12:56.580 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000001.11.5, , ] 2021-11-11T21:12:56.580 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:56.579+0000 7fe72a0ed380 0 bucket checkpoint complete 2021-11-11T21:12:56.599 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:56.600 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:56.600 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:56.651 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:56.651 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:56.714 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:56.715+0000 7fe9479ea380 1 bucket sync caught up with source: 2021-11-11T21:12:56.715 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.11.5, , ] 2021-11-11T21:12:56.715 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.11.5, , ] 2021-11-11T21:12:56.715 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:56.715+0000 7fe9479ea380 0 bucket checkpoint complete 2021-11-11T21:12:56.732 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:56.732 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:56.733 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:56.786 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:56.786 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:56.864 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:56.864+0000 7f8caa682380 1 bucket sync caught up with source: 2021-11-11T21:12:56.864 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000001.11.5, , ] 2021-11-11T21:12:56.865 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000001.11.5, , ] 2021-11-11T21:12:56.865 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:56.865+0000 7f8caa682380 0 bucket checkpoint complete 2021-11-11T21:12:56.880 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:56.881 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:56.882 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:56.931 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:56.932 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:57.003 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:57.003+0000 7fcb833fa380 1 bucket sync caught up with source: 2021-11-11T21:12:57.003 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.246.5, , ] 2021-11-11T21:12:57.003 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.246.5, , ] 2021-11-11T21:12:57.004 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:57.003+0000 7fcb833fa380 0 bucket checkpoint complete 2021-11-11T21:12:57.021 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:57.022 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:57.023 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:57.071 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:57.071 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:57.147 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:57.148+0000 7fcd16fec380 1 bucket sync caught up with source: 2021-11-11T21:12:57.147 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.246.5, , ] 2021-11-11T21:12:57.148 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.246.5, , ] 2021-11-11T21:12:57.148 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:57.148+0000 7fcd16fec380 0 bucket checkpoint complete 2021-11-11T21:12:57.164 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:57.165 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:57.165 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:57.213 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:57.214 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:57.271 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:57.272+0000 7f6a42d77380 1 bucket sync caught up with source: 2021-11-11T21:12:57.271 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.256.5, , ] 2021-11-11T21:12:57.272 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.256.5, , ] 2021-11-11T21:12:57.272 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:12:57.272+0000 7f6a42d77380 0 bucket checkpoint complete 2021-11-11T21:12:57.287 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:57.288 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:12:57.289 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:12:57.340 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:57.341 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:57.397 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:57.398+0000 7f0b60d6a380 1 bucket sync caught up with source: 2021-11-11T21:12:57.398 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000001.256.5, , ] 2021-11-11T21:12:57.398 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000001.256.5, , ] 2021-11-11T21:12:57.398 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:12:57.398+0000 7f0b60d6a380 0 bucket checkpoint complete 2021-11-11T21:12:57.413 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-20 zones={test-zone2, test-zone1} 2021-11-11T21:12:57.422 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:12:57.467 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:57.468 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:12:57.472 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:57.524 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:57.537 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:57.600 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:57.659 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-20 zones={test-zone2, test-zone1} 2021-11-11T21:12:57.660 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-20 zones={test-zone3, test-zone1} 2021-11-11T21:12:57.669 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:12:57.715 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:57.715 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:12:57.720 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:57.775 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:57.788 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:57.853 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:57.912 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-20 zones={test-zone3, test-zone1} 2021-11-11T21:12:57.912 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-20 zones={test-zone3, test-zone2} 2021-11-11T21:12:57.920 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:12:57.971 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:57.972 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:12:57.976 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:12:58.027 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:58.041 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:58.105 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:12:58.164 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-20 zones={test-zone3, test-zone2} 2021-11-11T21:12:58.174 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:58.175 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:58.176 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:58.219 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:58.220 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:58.287 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:12:58.304 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:12:58.305 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:58.305 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:58.305 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:12:58.305 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:58.306 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:58.306 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:12:58.306 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:12:58.306 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:58.306 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:58.307 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665178.174803_123.1", 2021-11-11T21:12:58.307 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:12:58.174803Z" 2021-11-11T21:12:58.307 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:12:58.308 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:12:58.308 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:58.308 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:12:58.308 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:12:58.309 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:12:58.310 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:12:58.310 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:58.311 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:12:58.312 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:12:58.356 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:12:58.356 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:12:58.436 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:12:58.436 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:12:58.436 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:12:58.437 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:12:58.437 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:12:58.437 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:12:58.437 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:58.438 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:58.438 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:12:58.438 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:58.439 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:12:58.439 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:58.439 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:58.439 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:12:58.439 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:58.440 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:12:58.440 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:58.440 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:12:58.440 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:58.441 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:58.441 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:58.441 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:58.441 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:12:58.441 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:58.442 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:58.442 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:12:58.442 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:58.442 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:58.443 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:58.443 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:12:58.443 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:58.443 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:58.443 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:58.444 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:58.444 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:12:58.444 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:58.444 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:58.444 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665178.169840_122.1", 2021-11-11T21:12:58.445 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:58.445 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:58.445 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:58.446 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:58.169840Z", 2021-11-11T21:12:58.446 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:58.446 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:58.446 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:58.447 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:12:58.447 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:12:58.447 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:12:58.447 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:12:58.447 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:12:58.448 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:12:58.448 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:12:58.448 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:12:58.448 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:12:58.449 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:12:58.449 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:58.449 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:58.449 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:12:58.449 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:12:58.450 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:12:58.450 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:12:58.450 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:12:58.450 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:12:58.451 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:12:58.452 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665178.174803_123.1 target=1_1636665178.169840_122.1 2021-11-11T21:13:03.453 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:03.454 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:03.455 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:03.503 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:03.503 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:03.581 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:03.581 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:03.582 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:03.582 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:03.582 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:03.582 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:03.583 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:03.583 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:03.583 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:03.583 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:03.584 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:03.584 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:03.584 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:03.584 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:03.585 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:03.585 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:03.585 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:03.585 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:03.585 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:03.586 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.586 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:03.586 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:03.586 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:03.587 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:03.587 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:03.587 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:03.587 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:03.588 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:03.588 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:03.588 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:03.588 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:03.588 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.589 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:03.589 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:03.589 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:03.589 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:03.590 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:03.590 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665178.174803_123.1", 2021-11-11T21:13:03.590 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:03.590 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:03.591 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:03.591 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:58.174803Z", 2021-11-11T21:13:03.591 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:03.591 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.591 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:03.592 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:03.592 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:03.592 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:03.592 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:03.593 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:03.593 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:03.593 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:03.593 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:03.594 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:03.594 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:03.594 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.594 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.595 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:03.595 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:03.595 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:03.595 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:03.596 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:03.596 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.597 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:03.598 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:13:03.599 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:13:03.599 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:03.600 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:03.600 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:03.647 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:03.647 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:03.718 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:13:03.719 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:13:03.719 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:13:03.719 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:13:03.719 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:13:03.720 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:03.720 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:03.720 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:03.720 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:13:03.721 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:03.721 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:13:03.721 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:03.721 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:03.721 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:03.722 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:03.722 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:13:03.722 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:03.722 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:03.723 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:03.723 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:03.723 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:03.723 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:03.724 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:13:03.724 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:03.724 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:03.724 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:13:03.725 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:03.725 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:03.725 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:03.725 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:13:03.725 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:03.726 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:03.726 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:03.726 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:03.726 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:13:03.727 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:03.727 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:03.727 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665178.174803_123.1", 2021-11-11T21:13:03.727 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:03.728 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:03.728 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:03.728 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:12:58.174803Z", 2021-11-11T21:13:03.728 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:03.728 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:03.729 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:03.729 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:03.729 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:13:03.729 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:03.730 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:03.730 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:03.730 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:03.730 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:03.731 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:03.731 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:03.731 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:03.731 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:03.732 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:03.732 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:13:03.732 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:03.732 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:13:03.733 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:13:03.733 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:13:03.733 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:03.733 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:13:03.735 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:13:03.735 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:13:03.736 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:03.736 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:03.737 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:03.780 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:03.780 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:03.847 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:03.847 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:03.847 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:03.847 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:03.848 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:03.848 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:03.848 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:03.848 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:03.849 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:03.849 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:03.849 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:03.849 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:03.850 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:03.850 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:03.850 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:03.850 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:03.851 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:03.851 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:03.851 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:03.851 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.852 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:03.852 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:03.852 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:03.852 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:03.853 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:03.853 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:03.853 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:03.853 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:03.854 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:03.854 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:03.854 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:03.854 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.855 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:03.855 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:03.855 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:03.855 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:03.856 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:03.856 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665178.174803_123.1", 2021-11-11T21:13:03.856 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:03.856 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:03.857 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:03.857 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:12:58.174803Z", 2021-11-11T21:13:03.857 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:03.857 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.857 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:03.858 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:03.858 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:03.858 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:03.858 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:03.859 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:03.859 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:03.859 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:03.859 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:03.860 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:03.860 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:03.860 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.860 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.860 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:03.861 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:03.861 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:03.861 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:03.861 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:03.862 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:03.862 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:03.863 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:13:03.911 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:03.912 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:03.912 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:03.961 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:03.962 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:04.020 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:04.021+0000 7f803d891380 1 waiting for incremental sync to catch up: 2021-11-11T21:13:04.021 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000001.11.5, , ] 2021-11-11T21:13:04.021 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000003.14.12, , ] 2021-11-11T21:13:09.022 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:09.023+0000 7f803d891380 1 bucket sync caught up with source: 2021-11-11T21:13:09.022 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000003.14.12, , ] 2021-11-11T21:13:09.023 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000003.14.12, , ] 2021-11-11T21:13:09.023 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:09.023+0000 7f803d891380 0 bucket checkpoint complete 2021-11-11T21:13:09.041 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.042 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.042 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:09.098 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:09.099 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:09.164 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:09.165+0000 7efdf999d380 1 bucket sync caught up with source: 2021-11-11T21:13:09.165 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000003.14.12, , ] 2021-11-11T21:13:09.165 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000003.14.12, , ] 2021-11-11T21:13:09.166 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:09.165+0000 7efdf999d380 0 bucket checkpoint complete 2021-11-11T21:13:09.184 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.185 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.186 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:09.241 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:09.242 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:09.307 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:09.309+0000 7ff6fd98a380 1 bucket sync caught up with source: 2021-11-11T21:13:09.308 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000003.14.12, , ] 2021-11-11T21:13:09.308 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000003.14.12, , ] 2021-11-11T21:13:09.308 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:09.309+0000 7ff6fd98a380 0 bucket checkpoint complete 2021-11-11T21:13:09.323 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.323 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.324 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:09.374 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:09.374 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:09.440 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:09.441+0000 7f132c198380 1 bucket sync caught up with source: 2021-11-11T21:13:09.441 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000004.261.9, , ] 2021-11-11T21:13:09.441 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000004.261.9, , ] 2021-11-11T21:13:09.441 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:09.441+0000 7f132c198380 0 bucket checkpoint complete 2021-11-11T21:13:09.462 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.463 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.463 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:09.511 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:09.512 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:09.578 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:09.579+0000 7f458c802380 1 bucket sync caught up with source: 2021-11-11T21:13:09.579 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000004.261.9, , ] 2021-11-11T21:13:09.579 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000004.261.9, , ] 2021-11-11T21:13:09.579 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:09.579+0000 7f458c802380 0 bucket checkpoint complete 2021-11-11T21:13:09.598 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.599 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.599 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:09.649 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:09.650 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:09.711 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:09.712+0000 7f5f669e8380 1 bucket sync caught up with source: 2021-11-11T21:13:09.712 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000004.271.9, , ] 2021-11-11T21:13:09.712 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000004.271.9, , ] 2021-11-11T21:13:09.712 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:09.712+0000 7f5f669e8380 0 bucket checkpoint complete 2021-11-11T21:13:09.727 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.728 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:09.728 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:09.781 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:09.782 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:09.845 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:09.846+0000 7f944ca96380 1 bucket sync caught up with source: 2021-11-11T21:13:09.846 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000004.271.9, , ] 2021-11-11T21:13:09.846 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000004.271.9, , ] 2021-11-11T21:13:09.846 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:09.846+0000 7f944ca96380 0 bucket checkpoint complete 2021-11-11T21:13:09.863 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-20 zones={test-zone2, test-zone1} 2021-11-11T21:13:09.872 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:09.919 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:09.920 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:09.920 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:09.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:09.925 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:09.980 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:09.994 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.058 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.116 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.129 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.199 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.255 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-20 zones={test-zone2, test-zone1} 2021-11-11T21:13:10.256 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-20 zones={test-zone3, test-zone1} 2021-11-11T21:13:10.264 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:10.311 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:10.312 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:10.312 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:10.318 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:10.318 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:10.368 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.381 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.446 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.507 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.521 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.586 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.643 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-20 zones={test-zone3, test-zone1} 2021-11-11T21:13:10.644 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-20 zones={test-zone3, test-zone2} 2021-11-11T21:13:10.653 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:10.699 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:10.700 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:10.700 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:10.705 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:10.705 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:10.755 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.769 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.839 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.899 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.911 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:10.973 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:11.031 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-20 zones={test-zone3, test-zone2} 2021-11-11T21:13:11.042 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:11.042 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:11.043 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:11.084 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:11.084 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:11.129 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:13:11.145 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:13:11.145 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:11.146 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:11.146 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:13:11.146 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:11.147 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:11.147 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:13:11.147 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:13:11.147 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:11.148 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:11.148 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665191.041655_125.1", 2021-11-11T21:13:11.148 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:13:11.041655Z" 2021-11-11T21:13:11.148 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:11.149 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:11.149 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:11.149 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:13:11.150 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:11.150 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:13:11.151 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:13:11.152 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:11.152 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:11.153 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:11.197 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:11.197 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:11.267 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:11.267 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:11.267 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:11.268 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:11.268 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:11.268 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:11.268 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:11.268 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:11.269 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:11.269 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:11.269 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:11.269 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:11.270 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:11.270 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:11.270 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:11.270 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:11.271 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:11.271 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:11.271 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:11.271 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:11.272 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:11.272 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:11.272 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:11.272 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:11.272 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:11.273 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:11.273 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:11.273 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:11.273 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:11.273 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:11.274 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:11.274 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:11.274 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:11.274 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:11.275 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:11.275 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:11.275 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:11.275 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665191.036635_124.1", 2021-11-11T21:13:11.275 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:11.276 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:11.276 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:11.276 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:11.036635Z", 2021-11-11T21:13:11.276 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:11.277 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:11.277 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:11.277 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:11.277 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:11.277 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:11.278 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:11.278 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:11.278 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:11.278 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:11.279 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:11.279 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:11.279 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:11.279 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:11.279 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:11.280 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:11.280 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:11.280 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:11.280 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:11.280 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:11.281 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:11.281 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:11.282 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665191.041655_125.1 target=1_1636665191.036635_124.1 2021-11-11T21:13:16.283 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:16.284 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:16.284 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:16.334 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:16.335 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:16.410 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:16.410 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:16.410 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:16.411 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:16.411 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:16.411 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:16.411 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:16.412 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:16.412 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:16.412 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:16.412 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:16.413 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:16.413 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:16.413 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:16.413 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:16.414 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:16.414 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:16.414 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:16.414 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:16.415 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.415 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:16.415 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:16.415 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:16.416 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:16.416 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:16.416 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:16.416 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:16.417 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:16.417 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:16.417 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:16.417 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:16.418 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.418 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:16.418 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:16.418 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:16.418 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:16.419 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:16.419 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665191.041655_125.1", 2021-11-11T21:13:16.419 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:16.419 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:16.420 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:16.420 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:11.041655Z", 2021-11-11T21:13:16.420 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:16.420 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.420 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:16.421 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:16.421 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:16.421 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:16.421 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:16.422 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:16.422 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:16.422 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:16.422 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:16.422 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:16.423 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:16.423 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.423 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.423 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:16.424 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:16.424 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:16.424 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:16.424 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:16.425 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.425 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:16.426 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:13:16.427 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:13:16.427 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:16.428 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:16.428 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:16.478 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:16.479 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:16.548 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:13:16.549 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:13:16.549 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:13:16.549 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:13:16.549 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:13:16.550 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:16.550 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:16.550 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:16.550 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:13:16.551 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:16.551 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:13:16.551 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:16.551 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:16.552 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:16.552 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:16.552 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:13:16.552 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:16.552 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:16.553 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:16.553 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:16.553 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:16.553 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:16.554 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:13:16.554 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:16.554 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:16.554 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:13:16.554 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:16.555 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:16.555 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:16.555 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:13:16.555 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:16.556 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:16.556 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:16.556 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:16.556 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:13:16.557 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:16.557 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:16.557 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665191.041655_125.1", 2021-11-11T21:13:16.557 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:16.558 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:16.558 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:16.558 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:13:11.041655Z", 2021-11-11T21:13:16.558 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:16.558 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:16.559 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:16.559 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:16.559 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:13:16.559 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:16.560 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:16.560 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:16.560 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:16.560 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:16.560 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:16.561 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:16.561 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:16.561 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:16.561 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:16.562 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:13:16.562 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:16.562 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:13:16.562 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:13:16.563 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:13:16.563 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:16.563 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:13:16.564 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:13:16.565 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:13:16.565 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:16.566 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:16.566 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:16.611 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:16.611 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:16.679 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:16.680 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:16.680 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:16.680 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:16.680 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:16.681 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:16.681 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:16.681 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:16.681 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:16.682 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:16.682 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:16.682 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:16.682 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:16.683 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:16.683 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:16.683 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:16.683 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:16.683 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:16.684 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:16.684 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.684 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:16.685 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:16.685 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:16.685 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:16.685 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:16.686 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:16.686 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:16.686 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:16.687 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:16.687 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:16.687 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:16.688 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.688 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:16.688 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:16.688 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:16.688 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:16.689 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:16.689 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665191.041655_125.1", 2021-11-11T21:13:16.689 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:16.689 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:16.690 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:16.690 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:11.041655Z", 2021-11-11T21:13:16.690 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:16.690 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.691 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:16.691 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:16.691 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:16.691 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:16.692 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:16.692 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:16.692 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:16.692 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:16.693 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:16.693 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:16.693 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:16.693 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.693 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.694 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:16.694 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:16.694 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:16.694 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:16.695 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:16.695 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:16.695 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:16.697 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:13:16.751 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:16.752 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:16.753 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:16.801 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:16.802 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:16.866 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:16.866+0000 7fc24746f380 1 waiting for incremental sync to catch up: 2021-11-11T21:13:16.866 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000003.14.12, , ] 2021-11-11T21:13:16.867 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000005.18.17, , ] 2021-11-11T21:13:21.868 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:21.868+0000 7fc24746f380 1 bucket sync caught up with source: 2021-11-11T21:13:21.868 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000005.18.17, , ] 2021-11-11T21:13:21.868 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000005.18.17, , ] 2021-11-11T21:13:21.869 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:21.868+0000 7fc24746f380 0 bucket checkpoint complete 2021-11-11T21:13:21.886 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:21.887 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:21.888 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:21.944 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:21.944 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:22.006 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:22.007+0000 7fdb82a1d380 1 bucket sync caught up with source: 2021-11-11T21:13:22.006 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000005.18.17, , ] 2021-11-11T21:13:22.007 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000005.18.17, , ] 2021-11-11T21:13:22.007 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:22.007+0000 7fdb82a1d380 0 bucket checkpoint complete 2021-11-11T21:13:22.023 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:22.024 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:22.025 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:22.080 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:22.081 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:22.143 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:22.144+0000 7f41b302a380 1 bucket sync caught up with source: 2021-11-11T21:13:22.144 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000005.18.17, , ] 2021-11-11T21:13:22.144 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000005.18.17, , ] 2021-11-11T21:13:22.144 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:22.144+0000 7f41b302a380 0 bucket checkpoint complete 2021-11-11T21:13:22.161 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:22.162 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:22.162 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:22.211 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:22.212 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:22.277 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:22.278+0000 7f4e9d1c6380 1 bucket sync caught up with source: 2021-11-11T21:13:22.278 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000007.277.14, , ] 2021-11-11T21:13:22.278 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000007.277.14, , ] 2021-11-11T21:13:22.278 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:22.278+0000 7f4e9d1c6380 0 bucket checkpoint complete 2021-11-11T21:13:22.302 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:22.303 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:22.304 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:22.356 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:22.357 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:22.422 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:22.423+0000 7f2640f85380 1 bucket sync caught up with source: 2021-11-11T21:13:22.423 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000007.277.14, , ] 2021-11-11T21:13:22.423 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000007.277.14, , ] 2021-11-11T21:13:22.423 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:22.423+0000 7f2640f85380 0 bucket checkpoint complete 2021-11-11T21:13:22.437 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:22.438 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:22.439 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:22.488 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:22.488 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:22.549 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:22.550+0000 7f9692071380 1 bucket sync caught up with source: 2021-11-11T21:13:22.550 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000007.287.14, , ] 2021-11-11T21:13:22.550 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000007.287.14, , ] 2021-11-11T21:13:22.551 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:22.550+0000 7f9692071380 0 bucket checkpoint complete 2021-11-11T21:13:22.572 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:22.572 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-20', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:22.573 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-20 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:22.626 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:22.627 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:22.704 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:22.704+0000 7f025ed3d380 1 bucket sync caught up with source: 2021-11-11T21:13:22.704 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000007.287.14, , ] 2021-11-11T21:13:22.704 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000007.287.14, , ] 2021-11-11T21:13:22.705 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:22.704+0000 7f025ed3d380 0 bucket checkpoint complete 2021-11-11T21:13:22.724 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-20 zones={test-zone2, test-zone1} 2021-11-11T21:13:22.733 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:22.789 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:22.790 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:22.790 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:22.796 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:22.797 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:22.852 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:22.865 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:22.940 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.000 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.013 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.081 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.139 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-20 zones={test-zone2, test-zone1} 2021-11-11T21:13:23.140 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-20 zones={test-zone3, test-zone1} 2021-11-11T21:13:23.148 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:23.195 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:23.196 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:23.196 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:23.201 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:23.201 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:23.256 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.270 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.334 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.391 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.404 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.465 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.523 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-20 zones={test-zone3, test-zone1} 2021-11-11T21:13:23.524 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-20 zones={test-zone3, test-zone2} 2021-11-11T21:13:23.532 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:23.583 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:23.584 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:23.584 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:23.589 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:23.589 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:23.644 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.660 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.730 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.791 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.805 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.871 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:23.927 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-20 zones={test-zone3, test-zone2} 2021-11-11T21:13:23.928 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_version_suspended_incremental_sync ... ok 2021-11-11T21:13:23.929 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-21 2021-11-11T21:13:23.949 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-22 2021-11-11T21:13:23.988 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-23 2021-11-11T21:13:24.063 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:24.064 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:24.065 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:24.107 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:24.108 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:24.160 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:13:24.177 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:13:24.178 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:24.178 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:24.178 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:13:24.178 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:24.179 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:24.179 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:13:24.179 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:13:24.179 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:24.180 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:24.180 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665204.061936_145.1", 2021-11-11T21:13:24.180 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:13:24.061936Z" 2021-11-11T21:13:24.180 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:24.180 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:24.181 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:24.181 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:13:24.181 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:24.181 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:13:24.182 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:13:24.183 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:24.184 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:24.184 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:24.227 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:24.227 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:24.291 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:24.291 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:24.291 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:24.292 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:24.292 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:24.292 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:24.292 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:24.293 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:24.293 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:24.293 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:24.293 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:24.294 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:24.294 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:24.294 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:24.294 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:24.295 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:24.295 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:24.295 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:24.295 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:24.296 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:24.296 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:24.296 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:24.296 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:24.297 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:24.297 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:24.297 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:24.297 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:24.298 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:24.298 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:24.298 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:24.298 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:24.299 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:24.299 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:24.299 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:24.299 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:24.300 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:24.300 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:24.300 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665203.939977_126.1", 2021-11-11T21:13:24.300 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:24.301 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:24.301 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:24.301 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:23.939977Z", 2021-11-11T21:13:24.301 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:24.302 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:24.302 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:24.302 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:24.302 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:24.302 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:24.303 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:24.303 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:24.303 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:24.303 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:24.304 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:24.304 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:24.304 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:24.304 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:24.305 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:24.305 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:24.305 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:24.305 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:24.306 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:24.306 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:24.306 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:24.306 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:24.308 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665204.061936_145.1 target=1_1636665203.939977_126.1 2021-11-11T21:13:29.309 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:29.310 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:29.311 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:29.359 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:29.359 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:29.431 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:29.431 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:29.432 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:29.432 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:29.432 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:29.432 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:29.433 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:29.433 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:29.433 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:29.433 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:29.433 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:29.434 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:29.434 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:29.434 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:29.434 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:29.435 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:29.435 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:29.435 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:29.435 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:29.436 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.436 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:29.436 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:29.436 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:29.437 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:29.437 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:29.437 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:29.437 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:29.437 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:29.438 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:29.438 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:29.438 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:29.438 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.439 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:29.439 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:29.439 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:29.439 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:29.440 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:29.440 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665204.061936_145.1", 2021-11-11T21:13:29.440 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:29.440 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:29.441 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:29.441 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:24.061936Z", 2021-11-11T21:13:29.441 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:29.441 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.441 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:29.442 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:29.442 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:29.442 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:29.442 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:29.443 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:29.443 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:29.443 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:29.443 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:29.444 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:29.444 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:29.444 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.444 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.444 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:29.445 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:29.445 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:29.445 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:29.445 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:29.446 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.446 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:29.447 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:13:29.448 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:13:29.448 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:29.449 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:29.450 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:29.498 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:29.499 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:29.571 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:13:29.572 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:13:29.572 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:13:29.572 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:13:29.572 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:13:29.573 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:29.573 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:29.573 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:29.573 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:13:29.574 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:29.574 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:13:29.574 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:29.574 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:29.574 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:29.575 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:29.575 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:13:29.575 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:29.576 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:29.576 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:29.576 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:29.577 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:29.577 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:29.577 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:13:29.577 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:29.578 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:29.578 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:13:29.578 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:29.578 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:29.579 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:29.579 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:13:29.579 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:29.579 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:29.579 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:29.580 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:29.580 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:13:29.580 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:29.580 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:29.581 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665204.061936_145.1", 2021-11-11T21:13:29.581 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:29.581 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:29.581 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:29.581 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:13:24.061936Z", 2021-11-11T21:13:29.582 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:29.582 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:29.582 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:29.582 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:29.583 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:13:29.583 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:29.583 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:29.583 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:29.584 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:29.584 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:29.584 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:29.584 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:29.585 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:29.585 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:29.585 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:29.585 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:13:29.585 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:29.586 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:13:29.586 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:13:29.586 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:13:29.586 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:29.587 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:13:29.588 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:13:29.588 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:13:29.589 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:29.589 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:29.590 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:29.634 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:29.634 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:29.700 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:29.701 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:29.701 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:29.701 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:29.702 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:29.702 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:29.702 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:29.702 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:29.703 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:29.703 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:29.703 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:29.703 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:29.704 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:29.704 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:29.704 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:29.704 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:29.705 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:29.705 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:29.705 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:29.705 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.705 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:29.706 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:29.706 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:29.706 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:29.706 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:29.707 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:29.707 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:29.707 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:29.707 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:29.708 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:29.708 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:29.708 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.708 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:29.708 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:29.709 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:29.709 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:29.709 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:29.709 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665204.061936_145.1", 2021-11-11T21:13:29.710 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:29.710 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:29.710 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:29.710 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:24.061936Z", 2021-11-11T21:13:29.711 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:29.711 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.711 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:29.711 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:29.712 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:29.712 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:29.712 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:29.712 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:29.713 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:29.713 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:29.713 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:29.713 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:29.714 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:29.714 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.714 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.714 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:29.714 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:29.715 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:29.715 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:29.715 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:29.715 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:29.716 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:29.717 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:13:29.914 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:29.915 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:29.915 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-21 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:29.968 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:29.968 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:30.048 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:30.049+0000 7f4b7534b380 1 bucket sync caught up with source: 2021-11-11T21:13:30.049 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000003.29.11, , ] 2021-11-11T21:13:30.049 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000003.29.11, , ] 2021-11-11T21:13:30.049 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:30.049+0000 7f4b7534b380 0 bucket checkpoint complete 2021-11-11T21:13:30.066 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.067 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.067 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-21 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:30.131 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:30.132 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:30.247 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:30.247+0000 7fdaa2ead380 1 bucket sync caught up with source: 2021-11-11T21:13:30.247 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000003.29.11, , ] 2021-11-11T21:13:30.247 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000003.29.11, , ] 2021-11-11T21:13:30.248 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:30.247+0000 7fdaa2ead380 0 bucket checkpoint complete 2021-11-11T21:13:30.272 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.273 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.273 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-21 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:30.328 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:30.328 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:30.396 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:30.396+0000 7fb8de155380 1 bucket sync caught up with source: 2021-11-11T21:13:30.396 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000003.29.11, , ] 2021-11-11T21:13:30.396 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000003.29.11, , ] 2021-11-11T21:13:30.397 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:30.396+0000 7fb8de155380 0 bucket checkpoint complete 2021-11-11T21:13:30.412 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.412 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.413 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-21 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:30.462 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:30.463 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:30.534 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:30.534+0000 7f8263e53380 1 bucket sync caught up with source: 2021-11-11T21:13:30.534 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000002.355.5, , ] 2021-11-11T21:13:30.534 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000002.355.5, , ] 2021-11-11T21:13:30.535 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:30.534+0000 7f8263e53380 0 bucket checkpoint complete 2021-11-11T21:13:30.549 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.550 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.551 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-21 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:30.602 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:30.602 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:30.674 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:30.675+0000 7f889625b380 1 bucket sync caught up with source: 2021-11-11T21:13:30.674 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000002.355.5, , ] 2021-11-11T21:13:30.675 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000002.355.5, , ] 2021-11-11T21:13:30.675 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:30.675+0000 7f889625b380 0 bucket checkpoint complete 2021-11-11T21:13:30.691 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.692 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.692 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-21 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:30.741 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:30.741 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:30.798 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:30.798+0000 7f6b2ab11380 1 bucket sync caught up with source: 2021-11-11T21:13:30.798 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000002.354.5, , ] 2021-11-11T21:13:30.799 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000002.354.5, , ] 2021-11-11T21:13:30.799 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:30.798+0000 7f6b2ab11380 0 bucket checkpoint complete 2021-11-11T21:13:30.819 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.820 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-21', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:30.820 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-21 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:30.872 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:30.873 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:30.942 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:30.943+0000 7fa184daf380 1 bucket sync caught up with source: 2021-11-11T21:13:30.943 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000002.354.5, , ] 2021-11-11T21:13:30.943 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000002.354.5, , ] 2021-11-11T21:13:30.943 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:30.943+0000 7fa184daf380 0 bucket checkpoint complete 2021-11-11T21:13:30.959 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-21 zones={test-zone2, test-zone1} 2021-11-11T21:13:30.969 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:31.019 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.020 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.020 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:31.025 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.026 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.084 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:31.092 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:31.107 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:31.174 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-21 zones={test-zone2, test-zone1} 2021-11-11T21:13:31.174 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-21 zones={test-zone3, test-zone1} 2021-11-11T21:13:31.183 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:31.235 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.235 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.236 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:31.241 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.241 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.296 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:31.303 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:31.316 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:31.382 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-21 zones={test-zone3, test-zone1} 2021-11-11T21:13:31.382 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-21 zones={test-zone3, test-zone2} 2021-11-11T21:13:31.390 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:31.439 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.440 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.440 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:31.446 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.446 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:31.495 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:31.503 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:31.515 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:31.578 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-21 zones={test-zone3, test-zone2} 2021-11-11T21:13:31.579 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:31.579 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:31.580 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-22 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:31.631 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:31.631 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:31.699 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:31.700+0000 7f5607750380 1 bucket sync caught up with source: 2021-11-11T21:13:31.700 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000002.9.5, , ] 2021-11-11T21:13:31.700 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000002.9.5, , ] 2021-11-11T21:13:31.700 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:31.700+0000 7f5607750380 0 bucket checkpoint complete 2021-11-11T21:13:31.715 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:31.715 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:31.716 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-22 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:31.765 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:31.765 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:31.835 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:31.835+0000 7f2df454b380 1 bucket sync caught up with source: 2021-11-11T21:13:31.835 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000002.9.5, , ] 2021-11-11T21:13:31.835 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000002.9.5, , ] 2021-11-11T21:13:31.836 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:31.835+0000 7f2df454b380 0 bucket checkpoint complete 2021-11-11T21:13:31.853 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:31.854 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:31.855 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-22 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:31.904 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:31.904 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:31.966 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:31.966+0000 7f4f1f604380 1 bucket sync caught up with source: 2021-11-11T21:13:31.967 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000002.9.5, , ] 2021-11-11T21:13:31.967 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000002.9.5, , ] 2021-11-11T21:13:31.968 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:31.966+0000 7f4f1f604380 0 bucket checkpoint complete 2021-11-11T21:13:31.982 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:31.982 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:31.983 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-22 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:32.033 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:32.033 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:32.103 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:32.104+0000 7fe9c259e380 1 bucket sync caught up with source: 2021-11-11T21:13:32.103 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000003.205.11, , ] 2021-11-11T21:13:32.104 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000003.205.11, , ] 2021-11-11T21:13:32.104 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:32.104+0000 7fe9c259e380 0 bucket checkpoint complete 2021-11-11T21:13:32.126 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:32.127 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:32.128 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-22 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:32.176 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:32.177 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:32.242 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:32.243+0000 7f04ef46b380 1 bucket sync caught up with source: 2021-11-11T21:13:32.243 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000003.205.11, , ] 2021-11-11T21:13:32.243 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000003.205.11, , ] 2021-11-11T21:13:32.243 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:32.243+0000 7f04ef46b380 0 bucket checkpoint complete 2021-11-11T21:13:32.262 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:32.263 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:32.263 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-22 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:32.313 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:32.314 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:32.378 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:32.379+0000 7f314ad87380 1 bucket sync caught up with source: 2021-11-11T21:13:32.379 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000002.205.5, , ] 2021-11-11T21:13:32.379 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000002.205.5, , ] 2021-11-11T21:13:32.380 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:32.379+0000 7f314ad87380 0 bucket checkpoint complete 2021-11-11T21:13:32.397 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:32.398 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-22', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:32.399 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-22 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:32.451 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:32.452 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:32.514 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:32.515+0000 7fc66d3a5380 1 bucket sync caught up with source: 2021-11-11T21:13:32.515 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000002.205.5, , ] 2021-11-11T21:13:32.515 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000002.205.5, , ] 2021-11-11T21:13:32.515 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:32.515+0000 7fc66d3a5380 0 bucket checkpoint complete 2021-11-11T21:13:32.530 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-22 zones={test-zone2, test-zone1} 2021-11-11T21:13:32.540 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:32.587 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:32.588 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:32.588 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:32.594 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:32.594 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:32.652 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:32.660 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:32.673 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:32.743 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-22 zones={test-zone2, test-zone1} 2021-11-11T21:13:32.743 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-22 zones={test-zone3, test-zone1} 2021-11-11T21:13:32.751 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:32.799 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:32.800 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:32.800 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:32.806 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:32.806 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:32.862 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:32.870 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:32.884 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:32.951 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-22 zones={test-zone3, test-zone1} 2021-11-11T21:13:32.951 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-22 zones={test-zone3, test-zone2} 2021-11-11T21:13:32.960 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:33.013 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:33.014 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:33.014 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:33.019 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:33.019 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:33.077 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:33.085 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:33.099 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:33.167 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-22 zones={test-zone3, test-zone2} 2021-11-11T21:13:33.167 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.168 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.169 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-23 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:33.218 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:33.219 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:33.289 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:33.290+0000 7f4e7230d380 1 bucket sync caught up with source: 2021-11-11T21:13:33.290 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000002.34.5, , ] 2021-11-11T21:13:33.290 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000002.34.5, , ] 2021-11-11T21:13:33.290 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:33.290+0000 7f4e7230d380 0 bucket checkpoint complete 2021-11-11T21:13:33.305 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.305 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.306 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-23 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:33.362 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:33.363 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:33.429 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:33.430+0000 7f493d1c8380 1 bucket sync caught up with source: 2021-11-11T21:13:33.430 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000002.34.5, , ] 2021-11-11T21:13:33.430 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000002.34.5, , ] 2021-11-11T21:13:33.430 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:33.430+0000 7f493d1c8380 0 bucket checkpoint complete 2021-11-11T21:13:33.448 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.449 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.450 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-23 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:33.501 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:33.502 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:33.559 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:33.560+0000 7fd05de70380 1 bucket sync caught up with source: 2021-11-11T21:13:33.560 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000002.34.5, , ] 2021-11-11T21:13:33.560 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000002.34.5, , ] 2021-11-11T21:13:33.560 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:33.560+0000 7fd05de70380 0 bucket checkpoint complete 2021-11-11T21:13:33.574 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.575 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.576 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-23 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:33.629 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:33.629 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:33.695 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:33.695+0000 7f3dcf4fd380 1 bucket sync caught up with source: 2021-11-11T21:13:33.695 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000002.500.5, , ] 2021-11-11T21:13:33.695 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000002.500.5, , ] 2021-11-11T21:13:33.696 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:33.695+0000 7f3dcf4fd380 0 bucket checkpoint complete 2021-11-11T21:13:33.713 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.714 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.715 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-23 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:33.762 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:33.763 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:33.834 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:33.835+0000 7f30d0051380 1 bucket sync caught up with source: 2021-11-11T21:13:33.835 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000002.500.5, , ] 2021-11-11T21:13:33.835 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000002.500.5, , ] 2021-11-11T21:13:33.835 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:33.835+0000 7f30d0051380 0 bucket checkpoint complete 2021-11-11T21:13:33.853 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.854 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.854 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-23 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:33.903 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:33.904 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:33.970 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:33.970+0000 7f01d23a8380 1 bucket sync caught up with source: 2021-11-11T21:13:33.970 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000003.510.11, , ] 2021-11-11T21:13:33.971 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000003.510.11, , ] 2021-11-11T21:13:33.971 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:33.970+0000 7f01d23a8380 0 bucket checkpoint complete 2021-11-11T21:13:33.988 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.989 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-23', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:33.990 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-23 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:34.040 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:34.040 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:34.103 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:34.103+0000 7fb4d0197380 1 bucket sync caught up with source: 2021-11-11T21:13:34.103 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000003.510.11, , ] 2021-11-11T21:13:34.104 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000003.510.11, , ] 2021-11-11T21:13:34.104 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:34.103+0000 7fb4d0197380 0 bucket checkpoint complete 2021-11-11T21:13:34.119 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-23 zones={test-zone2, test-zone1} 2021-11-11T21:13:34.127 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:34.179 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.180 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.180 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:34.185 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.186 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.235 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:34.243 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:34.258 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:34.330 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-23 zones={test-zone2, test-zone1} 2021-11-11T21:13:34.331 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-23 zones={test-zone3, test-zone1} 2021-11-11T21:13:34.342 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:34.387 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.388 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.388 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:34.393 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.393 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.449 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:34.456 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:34.469 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:34.534 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-23 zones={test-zone3, test-zone1} 2021-11-11T21:13:34.535 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-23 zones={test-zone3, test-zone2} 2021-11-11T21:13:34.544 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:34.591 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.592 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.592 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:34.597 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.598 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:34.648 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:34.656 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:34.668 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:34.734 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-23 zones={test-zone3, test-zone2} 2021-11-11T21:13:34.735 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_delete_marker_full_sync ... ok 2021-11-11T21:13:34.736 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-24 2021-11-11T21:13:34.759 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-25 2021-11-11T21:13:34.811 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-26 2021-11-11T21:13:34.923 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:34.923 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:34.924 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:34.966 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:34.966 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:35.028 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:13:35.043 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:13:35.043 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:35.044 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:35.044 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:13:35.044 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:35.044 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:35.045 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:13:35.045 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:13:35.045 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:35.045 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:35.045 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665214.920725_191.1", 2021-11-11T21:13:35.046 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:13:34.920725Z" 2021-11-11T21:13:35.046 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:35.046 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:35.046 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:35.047 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:13:35.047 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:35.047 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:13:35.048 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:13:35.049 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:35.049 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:35.050 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:35.092 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:35.093 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:35.155 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:35.155 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:35.156 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:35.156 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:35.156 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:35.156 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:35.157 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:35.157 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:35.157 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:35.157 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:35.157 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:35.158 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:35.158 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:35.158 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:35.158 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:35.159 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:35.159 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:35.159 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:35.159 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:35.160 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:35.160 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:35.160 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:35.160 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:35.161 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:35.161 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:35.161 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:35.161 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:35.162 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:35.162 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:35.162 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:35.162 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:35.163 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:35.163 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:35.163 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:35.163 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:35.163 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:35.164 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:35.164 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665214.748523_168.1", 2021-11-11T21:13:35.164 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:35.164 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:35.165 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:35.165 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:34.748523Z", 2021-11-11T21:13:35.165 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:35.165 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:35.165 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:35.166 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:35.166 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:35.166 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:35.166 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:35.167 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:35.167 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:35.167 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:35.168 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:35.168 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:35.168 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:35.168 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:35.168 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:35.169 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:35.169 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:35.169 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:35.169 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:35.170 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:35.170 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:35.170 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:35.171 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665214.920725_191.1 target=1_1636665214.748523_168.1 2021-11-11T21:13:40.172 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:40.173 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:40.174 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:40.223 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:40.223 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:40.293 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:40.293 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:40.294 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:40.294 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:40.294 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:40.294 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:40.295 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:40.295 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:40.295 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:40.295 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:40.296 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:40.296 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:40.296 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:40.296 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:40.297 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:40.297 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:40.297 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:40.297 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:40.297 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:40.298 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.298 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:40.298 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:40.298 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:40.299 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:40.299 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:40.299 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:40.299 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:40.300 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:40.300 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:40.300 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:40.300 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:40.300 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.301 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:40.301 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:40.301 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:40.301 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:40.302 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:40.302 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665214.920725_191.1", 2021-11-11T21:13:40.302 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:40.302 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:40.303 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:40.303 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:34.920725Z", 2021-11-11T21:13:40.303 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:40.303 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.303 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:40.304 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:40.304 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:40.304 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:40.304 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:40.305 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:40.305 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:40.305 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:40.305 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:40.305 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:40.306 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:40.306 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.306 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.306 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:40.307 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:40.307 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:40.307 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:40.308 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:40.308 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.308 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:40.309 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone2 2021-11-11T21:13:40.310 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone3 2021-11-11T21:13:40.310 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:40.311 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:40.312 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 metadata sync status --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:40.360 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:40.360 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:40.427 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:13:40.427 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": { 2021-11-11T21:13:40.427 INFO:teuthology.orchestra.run.smithi096.stdout: "info": { 2021-11-11T21:13:40.428 INFO:teuthology.orchestra.run.smithi096.stdout: "status": "sync", 2021-11-11T21:13:40.428 INFO:teuthology.orchestra.run.smithi096.stdout: "num_shards": 4, 2021-11-11T21:13:40.428 INFO:teuthology.orchestra.run.smithi096.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:40.429 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:40.430 INFO:teuthology.orchestra.run.smithi096.stdout: "markers": [ 2021-11-11T21:13:40.430 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:40.430 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 0, 2021-11-11T21:13:40.430 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:40.431 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:40.431 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:40.431 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:40.431 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 1, 2021-11-11T21:13:40.432 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:40.432 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:40.432 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:40.432 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:40.433 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:40.433 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:40.433 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 1, 2021-11-11T21:13:40.433 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:40.433 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:40.434 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:13:40.434 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:40.434 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:40.434 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:40.435 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "0.000000", 2021-11-11T21:13:40.435 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:40.435 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:40.435 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:40.435 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:40.436 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 2, 2021-11-11T21:13:40.436 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:40.437 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:40.437 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665214.920725_191.1", 2021-11-11T21:13:40.437 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:40.437 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:40.438 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:40.438 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:13:34.920725Z", 2021-11-11T21:13:40.438 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:40.439 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:40.439 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:40.439 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:40.439 INFO:teuthology.orchestra.run.smithi096.stdout: "key": 3, 2021-11-11T21:13:40.440 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:13:40.440 INFO:teuthology.orchestra.run.smithi096.stdout: "state": 1, 2021-11-11T21:13:40.440 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:40.441 INFO:teuthology.orchestra.run.smithi096.stdout: "next_step_marker": "", 2021-11-11T21:13:40.441 INFO:teuthology.orchestra.run.smithi096.stdout: "total_entries": 0, 2021-11-11T21:13:40.441 INFO:teuthology.orchestra.run.smithi096.stdout: "pos": 0, 2021-11-11T21:13:40.441 INFO:teuthology.orchestra.run.smithi096.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:40.442 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:13:40.442 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:40.442 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:40.442 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:13:40.443 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:40.443 INFO:teuthology.orchestra.run.smithi096.stdout: "full_sync": { 2021-11-11T21:13:40.443 INFO:teuthology.orchestra.run.smithi096.stdout: "total": 1, 2021-11-11T21:13:40.443 INFO:teuthology.orchestra.run.smithi096.stdout: "complete": 1 2021-11-11T21:13:40.444 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:40.444 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:13:40.445 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone3 2021-11-11T21:13:40.446 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone4 2021-11-11T21:13:40.446 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:40.447 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:40.447 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:40.491 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:40.491 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:40.558 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:40.559 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:40.559 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:40.559 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:40.559 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:40.560 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:40.560 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:40.560 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:40.560 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:40.561 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:40.561 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:40.561 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:40.561 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:40.561 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:40.562 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:40.562 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:40.562 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:40.562 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:40.563 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:40.563 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.563 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:40.563 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:40.563 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:40.564 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:40.564 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:40.564 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:40.564 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:40.565 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:40.565 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:40.565 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:40.565 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:40.565 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.566 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:40.566 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:40.566 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:40.566 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:40.567 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:40.567 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665214.920725_191.1", 2021-11-11T21:13:40.568 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:40.568 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:40.569 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:40.569 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:34.920725Z", 2021-11-11T21:13:40.569 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:40.569 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.569 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:40.570 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:40.570 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:40.570 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:40.570 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:40.571 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:40.571 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:40.571 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:40.571 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:40.572 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:40.572 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:40.572 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.572 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.572 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:40.573 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:40.573 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:40.573 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:40.573 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:40.574 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:40.574 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:40.575 INFO:rgw_multi.tests:finish meta checkpoint for zone=test-zone4 2021-11-11T21:13:40.788 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:40.789 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:40.789 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-24 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:40.841 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:40.871 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:40.926 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:40.925+0000 7fa6d1a06380 1 bucket sync caught up with source: 2021-11-11T21:13:40.926 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000003.19.6, , ] 2021-11-11T21:13:40.926 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000003.19.6, , ] 2021-11-11T21:13:40.927 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:40.925+0000 7fa6d1a06380 0 bucket checkpoint complete 2021-11-11T21:13:40.949 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:40.950 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:40.950 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-24 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:41.009 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:41.009 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:41.077 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:41.076+0000 7fef437cb380 1 bucket sync caught up with source: 2021-11-11T21:13:41.078 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000003.19.6, , ] 2021-11-11T21:13:41.078 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000003.19.6, , ] 2021-11-11T21:13:41.078 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:41.076+0000 7fef437cb380 0 bucket checkpoint complete 2021-11-11T21:13:41.095 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:41.096 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:41.096 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-24 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:41.147 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:41.147 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:41.209 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:41.210+0000 7f19ad95f380 1 bucket sync caught up with source: 2021-11-11T21:13:41.210 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000003.19.6, , ] 2021-11-11T21:13:41.210 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000003.19.6, , ] 2021-11-11T21:13:41.210 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:41.210+0000 7f19ad95f380 0 bucket checkpoint complete 2021-11-11T21:13:41.227 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:41.228 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:41.228 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-24 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:41.278 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:41.279 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:41.351 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:41.351+0000 7f811e473380 1 bucket sync caught up with source: 2021-11-11T21:13:41.351 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.213.12, , ] 2021-11-11T21:13:41.351 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.213.12, , ] 2021-11-11T21:13:41.351 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:41.351+0000 7f811e473380 0 bucket checkpoint complete 2021-11-11T21:13:41.375 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:41.376 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:41.377 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-24 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:41.425 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:41.426 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:41.487 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:41.488+0000 7f03cd8c2380 1 bucket sync caught up with source: 2021-11-11T21:13:41.488 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.213.12, , ] 2021-11-11T21:13:41.488 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.213.12, , ] 2021-11-11T21:13:41.488 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:41.488+0000 7f03cd8c2380 0 bucket checkpoint complete 2021-11-11T21:13:41.503 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:41.504 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:41.504 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-24 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:41.552 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:41.553 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:41.614 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:41.615+0000 7fcd58051380 1 bucket sync caught up with source: 2021-11-11T21:13:41.615 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.212.12, , ] 2021-11-11T21:13:41.615 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.212.12, , ] 2021-11-11T21:13:41.615 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:41.615+0000 7fcd58051380 0 bucket checkpoint complete 2021-11-11T21:13:41.630 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:41.631 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-24', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:41.631 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-24 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:41.682 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:41.683 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:41.745 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:41.745+0000 7f6395103380 1 bucket sync caught up with source: 2021-11-11T21:13:41.746 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000001.212.12, , ] 2021-11-11T21:13:41.746 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000001.212.12, , ] 2021-11-11T21:13:41.746 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:41.745+0000 7f6395103380 0 bucket checkpoint complete 2021-11-11T21:13:41.761 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-24 zones={test-zone2, test-zone1} 2021-11-11T21:13:41.770 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:41.819 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:41.820 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:41.825 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:41.876 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:41.882 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-24 zones={test-zone2, test-zone1} 2021-11-11T21:13:41.883 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-24 zones={test-zone3, test-zone1} 2021-11-11T21:13:41.891 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:41.939 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:41.940 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:41.945 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:41.995 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:42.002 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-24 zones={test-zone3, test-zone1} 2021-11-11T21:13:42.002 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-24 zones={test-zone3, test-zone2} 2021-11-11T21:13:42.010 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:42.059 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:42.060 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:42.064 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:42.119 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:42.127 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-24 zones={test-zone3, test-zone2} 2021-11-11T21:13:42.127 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.128 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.129 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-25 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:42.177 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:42.178 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:42.244 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:42.245+0000 7f6dddf14380 1 bucket sync caught up with source: 2021-11-11T21:13:42.245 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000001.11.12, , ] 2021-11-11T21:13:42.245 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000001.11.12, , ] 2021-11-11T21:13:42.245 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:42.245+0000 7f6dddf14380 0 bucket checkpoint complete 2021-11-11T21:13:42.262 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.263 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.264 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-25 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:42.314 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:42.314 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:42.381 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:42.382+0000 7f50ccf07380 1 bucket sync caught up with source: 2021-11-11T21:13:42.382 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.11.12, , ] 2021-11-11T21:13:42.382 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.11.12, , ] 2021-11-11T21:13:42.383 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:42.382+0000 7f50ccf07380 0 bucket checkpoint complete 2021-11-11T21:13:42.400 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.400 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.401 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-25 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:42.451 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:42.452 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:42.522 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:42.523+0000 7f6fc17f9380 1 bucket sync caught up with source: 2021-11-11T21:13:42.523 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000001.11.12, , ] 2021-11-11T21:13:42.523 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000001.11.12, , ] 2021-11-11T21:13:42.523 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:42.523+0000 7f6fc17f9380 0 bucket checkpoint complete 2021-11-11T21:13:42.537 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.537 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.538 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-25 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:42.587 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:42.587 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:42.657 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:42.658+0000 7f27e45ec380 1 bucket sync caught up with source: 2021-11-11T21:13:42.657 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000003.529.6, , ] 2021-11-11T21:13:42.658 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000003.529.6, , ] 2021-11-11T21:13:42.658 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:42.658+0000 7f27e45ec380 0 bucket checkpoint complete 2021-11-11T21:13:42.680 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.680 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.681 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-25 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:42.729 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:42.730 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:42.791 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:42.792+0000 7ff8ef5ea380 1 bucket sync caught up with source: 2021-11-11T21:13:42.792 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000003.529.6, , ] 2021-11-11T21:13:42.792 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000003.529.6, , ] 2021-11-11T21:13:42.792 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:42.792+0000 7ff8ef5ea380 0 bucket checkpoint complete 2021-11-11T21:13:42.808 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.808 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.809 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-25 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:42.861 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:42.861 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:42.934 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:42.934+0000 7f77fc383380 1 bucket sync caught up with source: 2021-11-11T21:13:42.934 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.275.12, , ] 2021-11-11T21:13:42.934 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.275.12, , ] 2021-11-11T21:13:42.934 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:42.934+0000 7f77fc383380 0 bucket checkpoint complete 2021-11-11T21:13:42.951 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.953 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-25', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:42.954 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-25 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:43.004 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:43.005 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:43.071 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:43.071+0000 7fb9cb386380 1 bucket sync caught up with source: 2021-11-11T21:13:43.071 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000001.275.12, , ] 2021-11-11T21:13:43.071 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000001.275.12, , ] 2021-11-11T21:13:43.072 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:43.071+0000 7fb9cb386380 0 bucket checkpoint complete 2021-11-11T21:13:43.088 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-25 zones={test-zone2, test-zone1} 2021-11-11T21:13:43.097 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:43.147 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:43.148 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:43.153 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:43.204 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:43.211 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-25 zones={test-zone2, test-zone1} 2021-11-11T21:13:43.212 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-25 zones={test-zone3, test-zone1} 2021-11-11T21:13:43.220 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:43.267 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:43.268 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:43.274 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:43.328 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:43.335 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-25 zones={test-zone3, test-zone1} 2021-11-11T21:13:43.335 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-25 zones={test-zone3, test-zone2} 2021-11-11T21:13:43.343 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:43.391 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:43.392 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:43.398 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:43.452 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:43.459 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-25 zones={test-zone3, test-zone2} 2021-11-11T21:13:43.459 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:43.460 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:43.461 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-26 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:43.510 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:43.510 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:43.583 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:43.583+0000 7f20a5056380 1 bucket sync caught up with source: 2021-11-11T21:13:43.583 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000001.37.12, , ] 2021-11-11T21:13:43.584 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000001.37.12, , ] 2021-11-11T21:13:43.584 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:43.583+0000 7f20a5056380 0 bucket checkpoint complete 2021-11-11T21:13:43.600 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:43.600 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:43.601 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-26 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:43.650 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:43.650 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:43.713 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:43.713+0000 7fb6df320380 1 bucket sync caught up with source: 2021-11-11T21:13:43.713 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.37.12, , ] 2021-11-11T21:13:43.713 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.37.12, , ] 2021-11-11T21:13:43.714 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:43.713+0000 7fb6df320380 0 bucket checkpoint complete 2021-11-11T21:13:43.728 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:43.729 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone1', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone4', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:43.730 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-26 --source-zone test-zone1 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone4 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:43.779 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:43.779 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:43.840 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:43.841+0000 7f1c20138380 1 bucket sync caught up with source: 2021-11-11T21:13:43.841 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000001.37.12, , ] 2021-11-11T21:13:43.841 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000001.37.12, , ] 2021-11-11T21:13:43.841 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:43.841+0000 7f1c20138380 0 bucket checkpoint complete 2021-11-11T21:13:43.856 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:43.857 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:43.857 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-26 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:43.906 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:43.906 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:43.973 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:43.974+0000 7fe594e38380 1 bucket sync caught up with source: 2021-11-11T21:13:43.974 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.362.12, , ] 2021-11-11T21:13:43.974 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.362.12, , ] 2021-11-11T21:13:43.974 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:43.974+0000 7fe594e38380 0 bucket checkpoint complete 2021-11-11T21:13:43.989 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:43.990 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone2', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone3', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:43.991 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-26 --source-zone test-zone2 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone3 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:44.039 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:44.039 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:44.097 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:44.098+0000 7fd69693f380 1 bucket sync caught up with source: 2021-11-11T21:13:44.098 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000001.362.12, , ] 2021-11-11T21:13:44.098 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000001.362.12, , ] 2021-11-11T21:13:44.098 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:44.098+0000 7fd69693f380 0 bucket checkpoint complete 2021-11-11T21:13:44.120 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:44.120 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:44.121 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 bucket sync checkpoint --bucket xdilif-26 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:44.170 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:44.171 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:44.241 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:44.241+0000 7f1084a07380 1 bucket sync caught up with source: 2021-11-11T21:13:44.241 INFO:teuthology.orchestra.run.smithi096.stderr: local status: [, , , , , , , , 00000000003.525.6, , ] 2021-11-11T21:13:44.242 INFO:teuthology.orchestra.run.smithi096.stderr: remote markers: [, , , , , , , , 00000000003.525.6, , ] 2021-11-11T21:13:44.242 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:13:44.241+0000 7f1084a07380 0 bucket checkpoint complete 2021-11-11T21:13:44.259 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:44.260 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'bucket', 'sync', 'checkpoint', '--bucket', 'xdilif-26', '--source-zone', 'test-zone3', '--retry-delay-ms', '5000', '--timeout-sec', '300', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '1', '--debug-ms', '0'] 2021-11-11T21:13:44.260 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 bucket sync checkpoint --bucket xdilif-26 --source-zone test-zone3 --retry-delay-ms 5000 --timeout-sec 300 --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 1 --debug-ms 0 2021-11-11T21:13:44.310 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:44.310 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:44.368 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:44.369+0000 7fd663511380 1 bucket sync caught up with source: 2021-11-11T21:13:44.369 INFO:teuthology.orchestra.run.smithi198.stderr: local status: [, , , , , , , , 00000000003.525.6, , ] 2021-11-11T21:13:44.369 INFO:teuthology.orchestra.run.smithi198.stderr: remote markers: [, , , , , , , , 00000000003.525.6, , ] 2021-11-11T21:13:44.369 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:13:44.369+0000 7fd663511380 0 bucket checkpoint complete 2021-11-11T21:13:44.384 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-26 zones={test-zone2, test-zone1} 2021-11-11T21:13:44.393 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:44.439 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:44.440 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:44.444 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:44.499 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:44.505 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-26 zones={test-zone2, test-zone1} 2021-11-11T21:13:44.506 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-26 zones={test-zone3, test-zone1} 2021-11-11T21:13:44.514 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:44.563 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:44.564 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:44.569 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:44.619 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:44.626 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-26 zones={test-zone3, test-zone1} 2021-11-11T21:13:44.626 INFO:tasks.rgw_multi.zone_rados:comparing bucket=xdilif-26 zones={test-zone3, test-zone2} 2021-11-11T21:13:44.634 DEBUG:tasks.rgw_multi.zone_rados:bucket1 objects: 2021-11-11T21:13:44.679 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:44.680 DEBUG:tasks.rgw_multi.zone_rados:bucket2 objects: 2021-11-11T21:13:44.684 DEBUG:tasks.rgw_multi.zone_rados:o=obj 2021-11-11T21:13:44.740 DEBUG:tasks.rgw_multi.zone_rados:comparing key name=obj 2021-11-11T21:13:44.746 INFO:tasks.rgw_multi.zone_rados:success, bucket identical: bucket=xdilif-26 zones={test-zone3, test-zone2} 2021-11-11T21:13:44.747 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_suspended_delete_marker_full_sync ... ok 2021-11-11T21:13:44.748 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-27 2021-11-11T21:13:44.769 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-28 2021-11-11T21:13:44.810 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-29 2021-11-11T21:13:45.031 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_versioning ... ok 2021-11-11T21:13:45.032 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-30 2021-11-11T21:13:45.056 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-31 2021-11-11T21:13:45.095 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-32 2021-11-11T21:13:45.363 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_acl ... ok 2021-11-11T21:13:45.364 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-33 2021-11-11T21:13:45.423 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-34 2021-11-11T21:13:45.554 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-35 2021-11-11T21:13:47.445 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:13:47.445+0000 7f04ef001700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-5) 2021-11-11T21:13:47.851 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_cors ... ok 2021-11-11T21:13:47.852 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-36 2021-11-11T21:13:47.885 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-37 2021-11-11T21:13:47.932 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-38 2021-11-11T21:13:47.971 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:47.972 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:47.972 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:48.014 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:48.014 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:48.067 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:13:48.083 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:13:48.083 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:48.084 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:48.084 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:13:48.084 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:48.084 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:48.085 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:13:48.085 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:13:48.085 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:48.085 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:48.086 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665227.956742_262.1", 2021-11-11T21:13:48.086 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:13:47.956742Z" 2021-11-11T21:13:48.086 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:13:48.086 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:13:48.086 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:48.087 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:13:48.087 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:13:48.087 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:13:48.088 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:13:48.089 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:48.089 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:48.090 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:48.133 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:48.134 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:48.200 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:48.201 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:48.201 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:48.201 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:48.202 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:48.202 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:48.202 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:48.202 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:48.203 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:48.203 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:48.203 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:48.203 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:48.203 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:48.204 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:48.204 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:48.204 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:48.204 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:48.205 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:48.205 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:48.205 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:48.205 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:48.206 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:48.206 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:48.206 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:48.206 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:48.207 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:48.207 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:48.207 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:48.207 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:48.208 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:48.208 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:48.208 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:48.208 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:48.209 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:48.209 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:48.209 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:48.209 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:48.209 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.055034_218.1", 2021-11-11T21:13:48.210 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:48.210 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:48.210 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:48.210 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.055034Z", 2021-11-11T21:13:48.211 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:48.211 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:48.211 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:48.211 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:48.212 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:48.212 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:48.212 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:48.212 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:48.213 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:48.213 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:48.213 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:48.213 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:48.214 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:48.214 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:48.214 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:48.214 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:48.214 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:48.215 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:48.215 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:48.215 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:48.215 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:48.216 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:48.218 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665225.055034_218.1 2021-11-11T21:13:53.219 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:53.220 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:53.220 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:53.263 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:53.263 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:53.334 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:53.335 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:53.335 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:53.335 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:53.336 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:53.336 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:53.337 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:53.337 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:53.338 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:53.338 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:53.338 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:53.339 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:53.339 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:53.340 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:53.340 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:53.341 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:53.341 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:53.341 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:53.342 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:53.342 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:53.343 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:53.343 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:53.344 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:53.344 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:53.344 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:53.345 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:53.346 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:53.346 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:53.346 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:53.347 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:53.347 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:53.348 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:53.348 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:53.349 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:53.349 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:53.349 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:53.350 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:53.350 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:13:53.351 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:53.351 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:53.352 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:53.352 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:13:53.352 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:53.353 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:53.353 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:53.354 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:53.354 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:53.355 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:53.355 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:53.355 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:53.356 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:53.356 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:53.357 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:53.357 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:53.357 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:53.358 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:53.358 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:53.359 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:53.359 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:53.360 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:53.360 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:53.360 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:53.361 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:53.361 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:53.363 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.762816_197.1 2021-11-11T21:13:58.365 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:58.366 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:13:58.366 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:13:58.408 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:13:58.408 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:13:58.479 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:13:58.479 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:13:58.480 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:13:58.480 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:13:58.480 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:13:58.480 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:13:58.480 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:58.481 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:58.481 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:13:58.481 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:58.481 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:13:58.482 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:58.482 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:58.482 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:13:58.482 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:58.483 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:13:58.483 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:58.483 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:13:58.483 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:58.483 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:58.484 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:58.484 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:58.484 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:13:58.484 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:58.485 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:58.485 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:13:58.485 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:58.485 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:58.485 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:58.486 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:13:58.486 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:58.486 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:58.486 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:58.487 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:58.487 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:13:58.487 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:58.487 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:58.488 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:13:58.488 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:58.488 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:58.488 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:58.488 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:13:58.489 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:58.489 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:58.489 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:58.489 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:13:58.490 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:13:58.490 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:13:58.490 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:13:58.490 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:13:58.490 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:13:58.491 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:13:58.491 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:13:58.491 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:13:58.491 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:13:58.491 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:58.492 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:58.492 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:13:58.492 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:13:58.492 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:13:58.493 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:13:58.493 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:13:58.493 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:13:58.493 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:13:58.494 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.831934_205.1 2021-11-11T21:14:03.496 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:03.497 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:03.498 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:14:03.616 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:14:03.617 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:14:04.048 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:14:04.048 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:14:04.048 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:14:04.049 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:14:04.049 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:14:04.049 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:14:04.049 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:04.050 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:04.050 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:14:04.050 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:04.050 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:14:04.051 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:04.051 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:04.051 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:14:04.051 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:04.052 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:14:04.052 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:04.052 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:14:04.052 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:04.053 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:04.053 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:04.053 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:04.053 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:14:04.053 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:04.054 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:04.054 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:14:04.054 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:04.054 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:04.055 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:04.055 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:14:04.055 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:04.055 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:04.056 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:04.056 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:04.056 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:14:04.056 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:04.057 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:04.057 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:14:04.057 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:04.057 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:04.057 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:04.058 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:14:04.058 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:04.058 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:04.058 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:04.059 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:04.059 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:14:04.059 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:04.059 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:04.059 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:14:04.060 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:04.060 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:04.060 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:04.060 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:14:04.061 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:04.061 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:04.061 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:04.061 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:14:04.062 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:04.062 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:14:04.062 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:14:04.062 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:14:04.063 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:04.063 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:14:04.063 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.762816_197.1 2021-11-11T21:14:09.064 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:09.065 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:09.066 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:14:09.108 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:14:09.109 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:14:09.177 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:14:09.178 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:14:09.178 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:14:09.178 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:14:09.178 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:14:09.179 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:14:09.179 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:09.179 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:09.179 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:14:09.180 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:09.180 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:14:09.180 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:09.180 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:09.181 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:14:09.181 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:09.181 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:14:09.181 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:09.182 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:14:09.182 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:09.182 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:09.182 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:09.182 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:09.183 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:14:09.183 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:09.183 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:09.183 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:14:09.184 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:09.184 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:09.184 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:09.184 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:14:09.185 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:09.185 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:09.185 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:09.185 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:09.186 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:14:09.186 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:09.186 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:09.186 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:14:09.187 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:09.187 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:09.187 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:09.187 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:14:09.187 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:09.188 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:09.188 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:09.188 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:09.188 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:14:09.189 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:09.189 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:09.189 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:14:09.189 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:09.189 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:09.190 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:09.190 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:14:09.190 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:09.190 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:09.191 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:09.191 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:14:09.191 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:09.191 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:14:09.192 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:14:09.192 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:14:09.192 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:09.192 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:14:09.194 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.783136_199.1 2021-11-11T21:14:14.195 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:14.196 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:14.196 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:14:14.241 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:14:14.241 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:14:14.310 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:14:14.310 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:14:14.310 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:14:14.311 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:14:14.311 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:14:14.311 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:14:14.311 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:14.312 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:14.312 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:14:14.312 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:14.312 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:14:14.313 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:14.313 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:14.313 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:14:14.313 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:14.313 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:14:14.314 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:14.314 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:14:14.314 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:14.314 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:14.314 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:14.315 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:14.315 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:14:14.315 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:14.315 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:14.316 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:14:14.316 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:14.316 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:14.316 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:14.316 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:14:14.317 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:14.317 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:14.317 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:14.317 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:14.318 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:14:14.318 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:14.318 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:14.318 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:14:14.319 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:14.319 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:14.319 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:14.319 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:14:14.319 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:14.320 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:14.320 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:14.320 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:14.320 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:14:14.321 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:14.321 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:14.321 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:14:14.321 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:14.321 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:14.322 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:14.322 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:14:14.322 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:14.322 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:14.323 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:14.323 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:14:14.323 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:14.323 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:14:14.323 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:14:14.324 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:14:14.324 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:14.324 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:14:14.325 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.787834_200.1 2021-11-11T21:14:19.326 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:19.327 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:19.328 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:14:19.376 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:14:19.376 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:14:19.449 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:14:19.449 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:14:19.449 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:14:19.450 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:14:19.450 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:14:19.450 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:14:19.450 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:19.451 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:19.451 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:14:19.451 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:19.451 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:14:19.452 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:19.452 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:19.452 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:14:19.452 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:19.452 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:14:19.453 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:19.453 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:14:19.453 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:19.453 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:19.454 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:19.454 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:19.454 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:14:19.454 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:19.454 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:19.455 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:14:19.455 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:19.455 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:19.455 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:19.456 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:14:19.456 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:19.456 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:19.456 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:19.457 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:19.457 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:14:19.457 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:19.457 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:19.457 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:14:19.458 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:19.458 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:19.458 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:19.458 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:14:19.459 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:19.459 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:19.459 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:19.459 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:19.459 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:14:19.460 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:19.460 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:19.460 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:14:19.460 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:19.461 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:19.461 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:19.461 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:14:19.461 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:19.462 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:19.462 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:19.462 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:14:19.462 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:19.462 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:14:19.463 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:14:19.463 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:14:19.463 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:19.463 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:14:19.465 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.762816_197.1 2021-11-11T21:14:24.466 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:24.467 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:24.468 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:14:24.512 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:14:24.512 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:14:24.580 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:14:24.580 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:14:24.581 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:14:24.581 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:14:24.581 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:14:24.581 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:14:24.582 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:24.582 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:24.582 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:14:24.582 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:24.583 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:14:24.583 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:24.583 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:24.583 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:14:24.584 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:24.584 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:14:24.584 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:24.584 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:14:24.585 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:24.585 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:24.585 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:24.585 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:24.585 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:14:24.586 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:24.586 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:24.586 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:14:24.586 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:24.587 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:24.587 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:24.587 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:14:24.587 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:24.588 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:24.588 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:24.588 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:24.588 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:14:24.589 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:24.589 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:24.589 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.049364_216.1", 2021-11-11T21:14:24.589 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:24.590 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:24.590 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:24.590 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.049364Z", 2021-11-11T21:14:24.590 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:24.590 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:24.591 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:24.591 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:24.591 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:14:24.591 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:24.592 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:24.592 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:14:24.592 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:24.592 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:24.593 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:24.593 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:14:24.593 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:24.593 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:24.593 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:24.594 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:14:24.594 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:24.594 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:14:24.594 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:14:24.595 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:14:24.595 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:24.595 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:14:24.596 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665225.049364_216.1 2021-11-11T21:14:29.598 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:29.599 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:29.600 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:14:29.648 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:14:29.648 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:14:29.723 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:14:29.724 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:14:29.724 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:14:29.724 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:14:29.724 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:14:29.725 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:14:29.725 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:29.725 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:29.725 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:14:29.726 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:29.726 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:14:29.726 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:29.726 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:29.727 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:14:29.727 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:29.727 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:14:29.727 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:29.727 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:14:29.728 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:29.728 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:29.728 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:29.728 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:29.729 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:14:29.729 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:29.729 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:29.729 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:14:29.730 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:29.730 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:29.730 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:29.730 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:14:29.730 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:29.731 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:29.731 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:29.731 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:29.731 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:14:29.732 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:29.732 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:29.732 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:14:29.732 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:29.732 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:29.733 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:29.733 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:14:29.733 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:29.733 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:29.734 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:29.734 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:29.734 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:14:29.734 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:29.735 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:29.735 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:14:29.735 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:29.735 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:29.736 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:29.736 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:14:29.736 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:29.736 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:29.736 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:29.737 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:14:29.737 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:29.737 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:14:29.737 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:14:29.737 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:14:29.738 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:29.738 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:14:29.739 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.762816_197.1 2021-11-11T21:14:34.740 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:34.741 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:34.742 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:14:34.786 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:14:34.786 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:14:34.853 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:14:34.854 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:14:34.854 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:14:34.854 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:14:34.855 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:14:34.855 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:14:34.855 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:34.856 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:34.856 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:14:34.856 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:34.856 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:14:34.857 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:34.857 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:34.857 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:14:34.857 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:34.857 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:14:34.858 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:34.858 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:14:34.858 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:34.858 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:34.859 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:34.859 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:34.859 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:14:34.859 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:34.859 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:34.860 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:14:34.860 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:34.860 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:34.860 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:34.861 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:14:34.861 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:34.861 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:34.861 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:34.862 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:34.862 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:14:34.862 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:34.862 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:34.862 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:14:34.863 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:34.863 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:34.863 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:34.863 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:14:34.864 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:34.864 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:34.864 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:34.864 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:34.864 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:14:34.871 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:34.872 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:34.872 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:14:34.872 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:34.872 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:34.873 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:34.873 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:14:34.873 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:34.873 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:34.874 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:34.874 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:14:34.874 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:34.874 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:14:34.874 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:14:34.875 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:14:34.875 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:34.875 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:14:34.877 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.831934_205.1 2021-11-11T21:14:39.878 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:39.879 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:39.880 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:14:39.922 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:14:39.922 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:14:39.986 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:14:39.987 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:14:39.987 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:14:39.987 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:14:39.987 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:14:39.988 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:14:39.988 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:39.988 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:39.988 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:14:39.988 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:39.989 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:14:39.989 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:39.989 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:39.989 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:14:39.990 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:39.990 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:14:39.990 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:39.990 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:14:39.991 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:39.991 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:39.991 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:39.991 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:39.992 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:14:39.992 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:39.992 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:39.992 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:14:39.992 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:39.993 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:39.993 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:39.993 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:14:39.993 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:39.994 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:39.994 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:39.994 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:39.994 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:14:39.995 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:39.995 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:39.995 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:14:39.995 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:39.995 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:39.996 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:39.996 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:14:39.996 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:39.996 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:39.997 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:39.997 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:39.997 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:14:39.997 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:39.998 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:39.998 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:14:39.998 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:39.998 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:39.999 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:39.999 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:14:39.999 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:39.999 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:40.000 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:40.000 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:14:40.000 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:40.000 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:14:40.000 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:14:40.001 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:14:40.001 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:40.001 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:14:40.002 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:14:45.004 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:45.005 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:45.005 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:14:45.047 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:14:45.048 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:14:45.116 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:14:45.116 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:14:45.116 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:14:45.117 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:14:45.117 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:14:45.117 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:14:45.118 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:45.119 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:45.119 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:14:45.120 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:45.120 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:14:45.121 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:45.121 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:45.121 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:14:45.122 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:45.122 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:14:45.123 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:45.123 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:14:45.123 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:45.124 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:45.124 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:45.125 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:45.125 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:14:45.125 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:45.126 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:45.126 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:14:45.126 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:45.126 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:45.127 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:45.127 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:14:45.127 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:45.128 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:45.128 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:45.128 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:45.128 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:14:45.128 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:45.129 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:45.129 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:14:45.129 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:45.129 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:45.130 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:45.130 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:14:45.130 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:45.130 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:45.131 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:45.131 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:45.131 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:14:45.131 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:45.131 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:45.132 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:14:45.132 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:45.132 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:45.132 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:45.133 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:14:45.133 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:45.133 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:45.133 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:45.133 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:14:45.134 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:45.134 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:14:45.134 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:14:45.134 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:14:45.135 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:45.135 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:14:45.136 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:14:50.138 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:50.139 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:50.139 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:14:50.191 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:14:50.191 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:14:50.257 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:14:50.258 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:14:50.258 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:14:50.258 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:14:50.259 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:14:50.259 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:14:50.259 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:50.259 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:50.260 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:14:50.260 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:50.260 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:14:50.261 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:50.261 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:50.261 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:14:50.261 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:50.261 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:14:50.262 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:50.262 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:14:50.262 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:50.262 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:50.263 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:50.263 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:50.263 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:14:50.264 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:50.264 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:50.264 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:14:50.265 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:50.265 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:50.265 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:50.266 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:14:50.266 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:50.266 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:50.266 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:50.266 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:50.267 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:14:50.267 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:50.267 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:50.267 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.114178_225.1", 2021-11-11T21:14:50.268 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:50.268 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:50.268 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:50.268 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.114178Z", 2021-11-11T21:14:50.269 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:50.269 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:50.269 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:50.269 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:50.270 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:14:50.270 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:50.270 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:50.270 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:14:50.271 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:50.271 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:50.271 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:50.271 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:14:50.272 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:50.272 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:50.272 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:50.272 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:14:50.272 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:50.273 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:14:50.273 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:14:50.273 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:14:50.273 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:50.274 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:14:50.275 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665225.114178_225.1 2021-11-11T21:14:55.277 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:55.278 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:14:55.278 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:14:55.320 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:14:55.320 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:14:55.390 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:14:55.391 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:14:55.391 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:14:55.392 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:14:55.393 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:14:55.393 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:14:55.394 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:55.394 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:55.395 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:14:55.395 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:55.396 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:14:55.396 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:55.396 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:55.397 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:14:55.397 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:55.398 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:14:55.398 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:55.399 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:14:55.399 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:55.400 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:55.400 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:55.400 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:55.401 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:14:55.401 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:55.402 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:55.402 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:14:55.403 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:55.403 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:55.404 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:55.404 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:14:55.405 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:55.405 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:55.405 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:55.406 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:55.406 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:14:55.407 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:55.407 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:55.408 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:14:55.408 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:55.409 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:55.410 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:55.410 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:14:55.411 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:55.411 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:55.412 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:55.412 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:14:55.413 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:14:55.413 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:14:55.414 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:14:55.414 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:14:55.415 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:14:55.415 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:14:55.416 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:14:55.416 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:14:55.417 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:14:55.417 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:55.418 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:55.418 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:14:55.419 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:14:55.419 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:14:55.420 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:14:55.420 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:14:55.421 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:14:55.421 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:14:55.423 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.761008_196.1 2021-11-11T21:15:00.425 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:00.427 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:00.428 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:00.512 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:00.513 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:00.577 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:00.577 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:00.578 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:00.578 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:00.578 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:00.578 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:00.579 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:00.579 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:00.579 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:00.580 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:00.580 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:00.580 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:00.580 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:00.581 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:00.581 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:00.581 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:00.581 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:00.582 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:00.582 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:00.582 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:00.582 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:00.583 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:00.583 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:00.583 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:00.584 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:00.584 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:00.584 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:00.584 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:00.585 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:00.585 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:00.585 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:00.585 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:00.586 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:00.586 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:00.586 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:00.586 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:00.587 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:00.587 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:15:00.587 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:00.588 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:00.588 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:00.588 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:15:00.589 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:00.589 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:00.589 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:00.589 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:00.590 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:00.590 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:00.590 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:00.591 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:00.591 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:00.591 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:00.591 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:00.592 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:00.592 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:00.592 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:00.593 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:00.593 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:00.593 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:00.594 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:00.594 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:00.594 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:00.594 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:00.595 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:00.596 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.787834_200.1 2021-11-11T21:15:05.597 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:05.598 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:05.598 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:05.673 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:05.674 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:05.768 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:05.768 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:05.769 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:05.769 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:05.770 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:05.770 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:05.771 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:05.771 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:05.772 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:05.772 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:05.772 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:05.773 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:05.774 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:05.775 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:05.776 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:05.776 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:05.777 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:05.777 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:05.778 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:05.778 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:05.779 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:05.779 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:05.780 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:05.780 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:05.781 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:05.781 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:05.782 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:05.782 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:05.782 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:05.783 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:05.783 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:05.784 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:05.784 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:05.785 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:05.785 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:05.786 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:05.786 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:05.787 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:15:05.787 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:05.788 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:05.788 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:05.789 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:15:05.789 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:05.789 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:05.790 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:05.790 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:05.791 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:05.791 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:05.792 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:05.794 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:05.794 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:05.795 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:05.795 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:05.796 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:05.796 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:05.796 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:05.796 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:05.797 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:05.798 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:05.798 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:05.798 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:05.798 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:05.799 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:05.799 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:05.800 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.831934_205.1 2021-11-11T21:15:10.802 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:10.803 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:10.804 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:10.850 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:10.850 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:10.921 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:10.922 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:10.922 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:10.922 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:10.923 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:10.924 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:10.924 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:10.925 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:10.925 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:10.926 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:10.926 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:10.927 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:10.927 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:10.928 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:10.929 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:10.929 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:10.930 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:10.930 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:10.931 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:10.931 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:10.931 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:10.932 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:10.932 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:10.933 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:10.933 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:10.934 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:10.935 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:10.935 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:10.935 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:10.936 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:10.936 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:10.937 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:10.937 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:10.938 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:10.938 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:10.939 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:10.939 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:10.939 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:15:10.940 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:10.940 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:10.941 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:10.941 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:15:10.942 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:10.942 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:10.942 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:10.943 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:10.943 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:10.944 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:10.945 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:10.945 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:10.945 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:10.946 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:10.947 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:10.947 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:10.948 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:10.948 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:10.949 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:10.949 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:10.949 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:10.950 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:10.950 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:10.951 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:10.952 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:10.952 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:10.954 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:15:15.955 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:15.956 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:15.957 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:16.022 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:16.022 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:16.124 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:16.125 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:16.125 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:16.125 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:16.125 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:16.126 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:16.126 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:16.126 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:16.126 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:16.127 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:16.128 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:16.128 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:16.129 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:16.129 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:16.129 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:16.129 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:16.130 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:16.130 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:16.130 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:16.130 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:16.130 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:16.131 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:16.131 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:16.131 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:16.131 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:16.132 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:16.132 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:16.132 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:16.132 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:16.132 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:16.133 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:16.133 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:16.133 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:16.134 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:16.135 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:16.135 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:16.135 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:16.136 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:15:16.136 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:16.136 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:16.136 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:16.137 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:15:16.137 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:16.137 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:16.137 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:16.137 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:16.138 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:16.138 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:16.138 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:16.138 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:16.139 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:16.139 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:16.139 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:16.139 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:16.140 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:16.140 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:16.140 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:16.140 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:16.140 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:16.141 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:16.141 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:16.141 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:16.141 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:16.142 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:16.143 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.831934_205.1 2021-11-11T21:15:21.144 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:21.145 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:21.145 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:21.187 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:21.188 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:21.251 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:21.251 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:21.251 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:21.252 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:21.252 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:21.252 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:21.252 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:21.253 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:21.253 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:21.253 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:21.254 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:21.254 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:21.254 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:21.255 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:21.255 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:21.255 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:21.256 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:21.256 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:21.256 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:21.256 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:21.257 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:21.257 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:21.257 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:21.257 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:21.257 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:21.258 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:21.258 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:21.258 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:21.258 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:21.259 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:21.259 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:21.259 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:21.260 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:21.260 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:21.260 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:21.260 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:21.260 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:21.261 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:15:21.261 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:21.262 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:21.262 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:21.262 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:15:21.263 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:21.263 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:21.264 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:21.264 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:21.264 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:21.264 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:21.265 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:21.265 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:21.265 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:21.265 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:21.266 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:21.266 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:21.266 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:21.266 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:21.267 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:21.267 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:21.267 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:21.267 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:21.267 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:21.268 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:21.268 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:21.268 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:21.270 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.761008_196.1 2021-11-11T21:15:22.315 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:15:22.315+0000 7fb6b95f5700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-5) 2021-11-11T21:15:26.271 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:26.272 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:26.273 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:26.316 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:26.317 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:26.448 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:26.449 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:26.449 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:26.449 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:26.449 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:26.450 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:26.450 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:26.450 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:26.451 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:26.451 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:26.452 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:26.452 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:26.452 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:26.452 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:26.453 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:26.453 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:26.453 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:26.453 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:26.454 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:26.454 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:26.454 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:26.454 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:26.455 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:26.455 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:26.455 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:26.455 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:26.455 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:26.456 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:26.456 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:26.456 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:26.456 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:26.457 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:26.457 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:26.457 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:26.457 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:26.457 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:26.458 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:26.458 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.050883_217.1", 2021-11-11T21:15:26.458 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:26.458 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:26.459 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:26.459 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.050883Z", 2021-11-11T21:15:26.459 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:26.459 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:26.460 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:26.460 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:26.460 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:26.460 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:26.461 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:26.461 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:26.461 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:26.461 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:26.461 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:26.462 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:26.462 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:26.462 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:26.462 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:26.462 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:26.463 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:26.463 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:26.463 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:26.463 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:26.463 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:26.464 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:26.465 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665225.050883_217.1 2021-11-11T21:15:31.466 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:31.467 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:31.469 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:31.513 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:31.514 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:31.583 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:31.584 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:31.585 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:31.585 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:31.586 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:31.586 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:31.586 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:31.586 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:31.587 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:31.587 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:31.588 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:31.588 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:31.588 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:31.588 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:31.589 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:31.589 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:31.589 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:31.589 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:31.590 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:31.590 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:31.590 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:31.590 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:31.591 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:31.591 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:31.591 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:31.591 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:31.592 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:31.592 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:31.592 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:31.592 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:31.592 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:31.593 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:31.593 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:31.593 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:31.593 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:31.594 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:31.594 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:31.594 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.794229_202.1", 2021-11-11T21:15:31.594 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:31.594 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:31.595 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:31.595 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.794229Z", 2021-11-11T21:15:31.595 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:31.596 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:31.596 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:31.596 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:31.596 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:31.597 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:31.597 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:31.597 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:31.597 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:31.597 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:31.598 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:31.598 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:31.598 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:31.598 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:31.599 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:31.599 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:31.599 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:31.599 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:31.599 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:31.600 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:31.600 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:31.600 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:31.601 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.794229_202.1 2021-11-11T21:15:36.603 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:36.604 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:36.605 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:36.678 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:36.679 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:36.816 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:36.817 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:36.817 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:36.817 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:36.818 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:36.818 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:36.818 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:36.818 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:36.819 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:36.819 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:36.819 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:36.819 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:36.820 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:36.820 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:36.820 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:36.820 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:36.820 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:36.821 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:36.821 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:36.821 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:36.821 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:36.822 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:36.822 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:36.822 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:36.822 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:36.823 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:36.823 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:36.823 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:36.823 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:36.824 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:36.824 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:36.824 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:36.824 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:36.824 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:36.825 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:36.825 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:36.825 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:36.825 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:15:36.826 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:36.826 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:36.826 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:36.826 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:15:36.827 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:36.827 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:36.827 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:36.827 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:36.828 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:36.828 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:36.828 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:36.828 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:36.828 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:36.829 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:36.829 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:36.829 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:36.829 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:36.830 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:36.830 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:36.830 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:36.830 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:36.831 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:36.831 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:36.831 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:36.831 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:36.831 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:36.833 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.761008_196.1 2021-11-11T21:15:41.834 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:41.835 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:41.836 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:41.904 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:41.905 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:41.986 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:41.986 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:41.986 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:41.987 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:41.987 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:41.987 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:41.987 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:41.987 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:41.988 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:41.988 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:41.988 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:41.988 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:41.989 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:41.989 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:41.989 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:41.989 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:41.990 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:41.990 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:41.990 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:41.990 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:41.990 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:41.991 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:41.991 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:41.991 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:41.991 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:41.992 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:41.992 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:41.992 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:41.992 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:41.992 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:41.993 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:41.993 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:41.993 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:41.993 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:41.993 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:41.994 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:41.994 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:41.994 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:15:41.994 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:41.995 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:41.995 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:41.995 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:15:41.995 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:41.995 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:41.996 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:41.996 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:41.996 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:41.996 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:41.996 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:41.997 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:41.997 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:41.997 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:41.997 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:41.997 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:41.998 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:41.998 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:41.998 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:41.998 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:41.999 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:41.999 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:41.999 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:41.999 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:41.999 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:42.000 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:42.001 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.762816_197.1 2021-11-11T21:15:42.426 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:15:42.426+0000 7fcab3609700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-5) 2021-11-11T21:15:47.002 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:47.003 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:47.003 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:47.070 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:47.070 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:47.183 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:47.183 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:47.183 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:47.184 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:47.184 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:47.184 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:47.184 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:47.185 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:47.185 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:47.185 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:47.185 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:47.185 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:47.186 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:47.186 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:47.186 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:47.186 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:47.187 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:47.187 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:47.187 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:47.187 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:47.188 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:47.188 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:47.188 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:47.188 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:47.188 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:47.189 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:47.189 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:47.189 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:47.189 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:47.189 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:47.190 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:47.190 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:47.190 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:47.190 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:47.191 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:47.191 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:47.191 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:47.191 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:15:47.192 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:47.192 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:47.192 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:47.192 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:15:47.192 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:47.193 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:47.193 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:47.193 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:47.193 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:47.194 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:47.194 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:47.194 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:47.194 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:47.194 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:47.195 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:47.195 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:47.195 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:47.195 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:47.195 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:47.196 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:47.196 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:47.196 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:47.196 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:47.197 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:47.197 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:47.197 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:47.198 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.762816_197.1 2021-11-11T21:15:52.199 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:52.200 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:52.201 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:52.288 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:52.289 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:52.388 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:52.389 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:52.389 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:52.389 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:52.390 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:52.390 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:52.390 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:52.390 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:52.391 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:52.391 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:52.391 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:52.391 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:52.392 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:52.392 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:52.392 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:52.392 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:52.393 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:52.393 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:52.393 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:52.393 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:52.394 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:52.394 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:52.394 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:52.394 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:52.395 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:52.395 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:52.395 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:52.395 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:52.395 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:52.396 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:52.396 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:52.396 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:52.396 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:52.397 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:52.397 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:52.397 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:52.397 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:52.398 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:15:52.398 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:52.398 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:52.398 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:52.398 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:15:52.399 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:52.399 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:52.399 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:52.399 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:52.400 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:52.400 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:52.400 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:52.400 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:52.401 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:52.401 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:52.401 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:52.401 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:52.402 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:52.402 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:52.402 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:52.402 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:52.402 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:52.403 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:52.403 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:52.403 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:52.403 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:52.404 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:52.405 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.787834_200.1 2021-11-11T21:15:57.406 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:57.407 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:15:57.408 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:15:57.451 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:15:57.451 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:15:57.523 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:15:57.523 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:15:57.523 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:15:57.524 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:15:57.524 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:15:57.524 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:15:57.524 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:57.525 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:57.525 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:15:57.525 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:57.525 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:15:57.526 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:57.526 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:57.526 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:15:57.526 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:57.527 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:15:57.527 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:57.527 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:15:57.527 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:57.527 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:57.528 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:57.528 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:57.528 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:15:57.528 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:57.529 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:57.529 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:15:57.529 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:57.530 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:57.530 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:57.530 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:15:57.530 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:57.531 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:57.531 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:57.531 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:57.531 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:15:57.532 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:57.532 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:57.532 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.050883_217.1", 2021-11-11T21:15:57.533 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:57.533 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:57.533 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:57.533 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.050883Z", 2021-11-11T21:15:57.533 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:57.534 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:57.534 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:57.534 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:15:57.534 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:15:57.535 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:15:57.535 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:15:57.535 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:15:57.535 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:15:57.536 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:15:57.536 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:15:57.536 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:15:57.536 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:15:57.536 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:57.537 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:57.537 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:15:57.537 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:15:57.537 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:15:57.538 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:15:57.538 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:15:57.538 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:15:57.538 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:15:57.540 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665225.050883_217.1 2021-11-11T21:16:02.541 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:02.542 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:02.543 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:02.608 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:02.609 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:02.679 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:02.680 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:02.680 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:02.680 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:02.680 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:02.681 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:02.681 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:02.681 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:02.681 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:02.682 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:02.682 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:02.682 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:02.682 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:02.683 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:02.683 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:02.683 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:02.683 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:02.683 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:02.684 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:02.684 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:02.684 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:02.684 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:02.685 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:02.685 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:02.685 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:02.685 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:02.686 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:02.686 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:02.686 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:02.687 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:02.687 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:02.687 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:02.687 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:02.688 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:02.688 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:02.688 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:02.688 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:02.689 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:16:02.689 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:02.689 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:02.689 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:02.690 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:16:02.690 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:02.690 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:02.690 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:02.691 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:02.691 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:02.691 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:02.691 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:02.692 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:02.692 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:02.692 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:02.692 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:02.692 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:02.693 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:02.693 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:02.693 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:02.693 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:02.694 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:02.694 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:02.694 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:02.694 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:02.695 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:02.695 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:02.697 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.825641_203.1 2021-11-11T21:16:07.698 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:07.699 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:07.699 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:07.742 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:07.743 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:07.814 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:07.814 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:07.815 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:07.815 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:07.815 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:07.816 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:07.816 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:07.816 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:07.816 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:07.816 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:07.817 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:07.817 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:07.817 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:07.817 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:07.818 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:07.818 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:07.818 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:07.818 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:07.819 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:07.819 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:07.819 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:07.819 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:07.820 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:07.820 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:07.820 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:07.820 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:07.820 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:07.821 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:07.821 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:07.821 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:07.821 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:07.822 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:07.822 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:07.822 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:07.822 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:07.823 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:07.823 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:07.823 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:16:07.823 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:07.824 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:07.824 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:07.824 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:16:07.824 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:07.825 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:07.825 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:07.825 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:07.825 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:07.825 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:07.826 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:07.826 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:07.826 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:07.826 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:07.827 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:07.827 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:07.827 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:07.827 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:07.828 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:07.828 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:07.828 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:07.828 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:07.828 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:07.829 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:07.829 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:07.829 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:07.831 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.831934_205.1 2021-11-11T21:16:12.832 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:12.833 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:12.833 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:12.888 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:12.889 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:12.961 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:12.961 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:12.961 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:12.961 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:12.962 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:12.962 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:12.962 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:12.962 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:12.963 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:12.963 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:12.963 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:12.963 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:12.964 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:12.964 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:12.964 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:12.964 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:12.964 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:12.965 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:12.965 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:12.965 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:12.965 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:12.966 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:12.966 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:12.966 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:12.966 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:12.966 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:12.968 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:12.968 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:12.968 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:12.968 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:16:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:12.970 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:12.970 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:12.970 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:16:12.970 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:12.970 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:12.971 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:12.971 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:12.971 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:12.971 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:12.972 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:12.972 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:12.972 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:12.972 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:12.972 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:12.973 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:12.973 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:12.973 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:12.973 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:12.974 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:12.975 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:12.975 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:12.976 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:12.976 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:12.976 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:12.976 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:12.978 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665225.075927_221.1 2021-11-11T21:16:17.979 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:17.980 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:17.981 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:18.027 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:18.027 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:18.095 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:18.095 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:18.095 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:18.096 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:18.096 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:18.096 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:18.096 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:18.097 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:18.097 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:18.097 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:18.097 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:18.097 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:18.098 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:18.098 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:18.098 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:18.098 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:18.099 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:18.099 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:18.099 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:18.099 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:18.100 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:18.100 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:18.100 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:18.100 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:18.100 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:18.101 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:18.101 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:18.101 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:18.101 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:18.102 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:18.102 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:18.102 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:18.102 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:18.102 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:18.103 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:18.103 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:18.103 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:18.103 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:16:18.104 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:18.104 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:18.104 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:18.104 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:16:18.104 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:18.105 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:18.105 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:18.105 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:18.105 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:18.106 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:18.106 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:18.106 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:18.106 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:18.106 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:18.107 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:18.107 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:18.107 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:18.107 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:18.108 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:18.108 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:18.108 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:18.108 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:18.108 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:18.109 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:18.109 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:18.109 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:18.110 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.762816_197.1 2021-11-11T21:16:23.111 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:23.112 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:23.112 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:23.156 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:23.157 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:23.224 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:23.224 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:23.225 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:23.225 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:23.225 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:23.225 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:23.226 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:23.226 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:23.226 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:23.226 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:23.227 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:23.227 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:23.227 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:23.227 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:23.227 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:23.228 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:23.228 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:23.228 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:23.228 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:23.229 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:23.229 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:23.229 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:23.229 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:23.230 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:23.230 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:23.230 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:23.230 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:23.230 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:23.231 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:23.231 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:23.231 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:23.231 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:23.232 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:23.232 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:23.232 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:23.232 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:23.233 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:23.233 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:16:23.233 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:23.233 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:23.233 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:23.234 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:16:23.234 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:23.234 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:23.234 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:23.234 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:23.235 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:23.235 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:23.235 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:23.235 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:23.235 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:23.236 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:23.236 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:23.236 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:23.236 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:23.237 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:23.237 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:23.237 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:23.237 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:23.237 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:23.238 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:23.238 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:23.238 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:23.238 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:23.240 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:16:28.242 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:28.242 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:28.243 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:28.286 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:28.286 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:28.364 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:28.364 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:28.364 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:28.365 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:28.365 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:28.365 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:28.365 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:28.365 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:28.366 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:28.366 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:28.366 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:28.366 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:28.367 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:28.367 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:28.367 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:28.367 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:28.368 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:28.368 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:28.368 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:28.368 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:28.369 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:28.369 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:28.369 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:28.369 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:28.369 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:28.370 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:28.370 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:28.370 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:28.370 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:28.371 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:28.371 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:28.371 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:28.371 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:28.372 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:28.372 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:28.372 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:28.372 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:28.372 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:16:28.373 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:28.373 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:28.373 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:28.373 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:16:28.374 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:28.374 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:28.374 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:28.374 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:28.375 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:28.375 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:28.375 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:28.375 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:28.375 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:28.376 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:28.376 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:28.376 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:28.376 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:28.377 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:28.377 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:28.377 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:28.377 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:28.377 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:28.378 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:28.378 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:28.378 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:28.378 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:28.380 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.825641_203.1 2021-11-11T21:16:33.381 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:33.382 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:33.382 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:33.425 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:33.425 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:33.495 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:33.495 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:33.496 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:33.496 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:33.496 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:33.496 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:33.497 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:33.497 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:33.497 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:33.497 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:33.498 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:33.498 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:33.498 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:33.498 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:33.498 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:33.499 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:33.499 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:33.499 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:33.499 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:33.500 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:33.500 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:33.500 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:33.500 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:33.500 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:33.501 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:33.501 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:33.501 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:33.501 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:33.502 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:33.502 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:33.502 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:33.502 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:33.503 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:33.503 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:33.503 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:33.503 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:33.504 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:33.504 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:16:33.504 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:33.504 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:33.504 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:33.505 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:16:33.505 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:33.505 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:33.505 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:33.506 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:33.506 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:33.506 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:33.506 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:33.507 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:33.507 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:33.507 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:33.507 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:33.507 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:33.508 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:33.508 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:33.508 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:33.508 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:33.509 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:33.509 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:33.509 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:33.509 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:33.509 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:33.510 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:33.511 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.761008_196.1 2021-11-11T21:16:38.513 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:38.513 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:38.514 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:38.556 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:38.557 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:38.628 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:38.628 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:38.629 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:38.629 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:38.629 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:38.629 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:38.630 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:38.630 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:38.630 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:38.630 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:38.631 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:38.631 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:38.631 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:38.631 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:38.632 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:38.632 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:38.632 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:38.632 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:38.632 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:38.633 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:38.633 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:38.633 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:38.633 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:38.633 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:38.634 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:38.634 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:38.634 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:38.634 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:38.635 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:38.635 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:38.635 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:38.635 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:38.636 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:38.636 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:38.636 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:38.636 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:38.636 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:38.637 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:16:38.637 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:38.637 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:38.637 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:38.637 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:16:38.638 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:38.638 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:38.638 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:38.638 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:38.639 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:38.639 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:38.639 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:38.639 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:38.639 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:38.640 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:38.640 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:38.640 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:38.640 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:38.640 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:38.641 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:38.641 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:38.641 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:38.641 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:38.641 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:38.642 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:38.642 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:38.642 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:38.643 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.761008_196.1 2021-11-11T21:16:43.644 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:43.645 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:43.646 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:43.691 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:43.691 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:43.767 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:43.767 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:43.767 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:43.767 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:43.768 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:43.768 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:43.768 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:43.768 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:43.768 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:43.769 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:43.769 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:43.769 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:43.769 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:43.769 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:43.770 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:43.770 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:43.770 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:43.770 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:43.771 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:43.771 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:43.771 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:43.771 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:43.771 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:43.772 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:43.772 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:43.772 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:43.772 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:43.772 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:43.773 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:43.773 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:43.773 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:43.773 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:43.774 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:43.774 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:43.774 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:43.774 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:43.774 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:43.775 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:16:43.775 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:43.775 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:43.775 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:43.776 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:16:43.776 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:43.776 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:43.776 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:43.776 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:43.777 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:43.777 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:43.777 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:43.777 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:43.777 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:43.778 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:43.778 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:43.778 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:43.778 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:43.778 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:43.779 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:43.779 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:43.779 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:43.779 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:43.779 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:43.780 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:43.780 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:43.780 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:43.781 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.830110_204.1 2021-11-11T21:16:48.782 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:48.783 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:48.784 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:48.829 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:48.829 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:48.901 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:48.901 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:48.901 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:48.902 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:48.902 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:48.902 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:48.902 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:48.903 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:48.903 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:48.903 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:48.903 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:48.904 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:48.904 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:48.904 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:48.904 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:48.905 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:48.905 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:48.905 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:48.905 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:48.905 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:48.906 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:48.906 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:48.906 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:48.907 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:48.907 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:48.907 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:48.907 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:48.908 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:48.908 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:48.908 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:48.908 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:48.908 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:48.909 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:48.909 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:48.909 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:48.909 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:48.910 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:48.910 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:16:48.910 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:48.910 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:48.910 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:48.911 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:16:48.911 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:48.911 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:48.911 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:48.912 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:48.912 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:48.912 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:48.912 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:48.912 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:48.913 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:48.913 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:48.913 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:48.913 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:48.914 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:48.914 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:48.914 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:48.914 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:48.915 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:48.915 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:48.915 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:48.915 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:48.915 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:48.916 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:48.917 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.831934_205.1 2021-11-11T21:16:53.918 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:53.919 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:53.920 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:53.962 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:53.962 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:54.031 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:54.031 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:54.032 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:54.032 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:54.032 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:54.032 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:54.033 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:54.033 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:54.033 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:54.033 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:54.034 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:54.034 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:54.034 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:54.034 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:54.035 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:54.035 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:54.035 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:54.035 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:54.035 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:54.036 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:54.036 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:54.036 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:54.036 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:54.037 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:54.037 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:54.037 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:54.037 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:54.037 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:54.038 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:54.038 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:54.038 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:54.038 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:54.039 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:54.039 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:54.039 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:54.039 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:54.039 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:54.040 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.794229_202.1", 2021-11-11T21:16:54.040 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:54.040 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:54.040 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:54.040 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.794229Z", 2021-11-11T21:16:54.041 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:54.041 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:54.041 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:54.041 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:54.042 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:54.042 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:54.042 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:54.042 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:54.042 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:54.043 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:54.043 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:54.043 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:54.043 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:54.044 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:54.044 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:54.044 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:54.044 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:54.044 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:54.045 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:54.045 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:54.045 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:54.045 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:54.046 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.794229_202.1 2021-11-11T21:16:59.047 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:59.049 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:16:59.049 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:16:59.092 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:16:59.093 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:16:59.164 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:16:59.165 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:16:59.165 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:16:59.165 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:16:59.165 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:16:59.166 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:16:59.166 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:59.166 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:59.166 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:16:59.167 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:59.167 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:16:59.167 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:59.167 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:59.168 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:16:59.168 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:59.168 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:16:59.168 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:59.169 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:16:59.169 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:59.169 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:59.169 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:59.169 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:59.170 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:16:59.170 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:59.170 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:59.170 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:16:59.171 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:59.171 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:59.171 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:59.171 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:16:59.171 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:59.172 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:59.172 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:59.172 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:59.172 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:16:59.173 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:59.173 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:59.173 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.794229_202.1", 2021-11-11T21:16:59.173 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:59.173 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:59.174 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:59.174 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.794229Z", 2021-11-11T21:16:59.174 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:59.174 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:59.175 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:59.175 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:16:59.175 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:16:59.175 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:16:59.176 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:16:59.176 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:16:59.176 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:16:59.176 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:16:59.176 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:16:59.177 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:16:59.177 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:16:59.177 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:59.177 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:59.177 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:16:59.178 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:16:59.178 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:16:59.178 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:16:59.179 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:16:59.179 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:16:59.179 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:16:59.180 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.794229_202.1 2021-11-11T21:17:04.181 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:04.182 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:04.182 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:17:04.230 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:17:04.231 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:17:04.299 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:17:04.300 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:17:04.300 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:17:04.300 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:17:04.300 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:17:04.301 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:17:04.301 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:04.301 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:04.301 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:17:04.302 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:04.302 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:17:04.302 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:04.302 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:04.302 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:17:04.303 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:04.303 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:17:04.303 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:04.303 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:17:04.303 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:04.304 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:04.304 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:04.304 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:04.304 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:17:04.304 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:04.305 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:04.305 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:17:04.305 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:04.305 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:04.305 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:04.306 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:17:04.306 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:04.306 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:04.306 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:04.307 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:04.307 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:17:04.307 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:04.307 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:04.307 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:17:04.308 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:04.308 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:04.308 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:04.308 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:17:04.308 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:04.309 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:04.309 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:04.309 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:04.309 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:17:04.309 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:04.310 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:04.310 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:17:04.310 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:04.310 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:04.311 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:04.311 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:17:04.311 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:04.311 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:04.311 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:04.312 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:17:04.312 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:04.312 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:17:04.312 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:17:04.312 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:17:04.313 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:04.313 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:17:04.314 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:17:09.315 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:09.316 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:09.316 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:17:09.359 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:17:09.360 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:17:09.440 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:17:09.440 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:17:09.441 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:17:09.441 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:17:09.441 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:17:09.441 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:17:09.442 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:09.442 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:09.442 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:17:09.442 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:09.443 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:17:09.443 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:09.443 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:09.443 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:17:09.444 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:09.444 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:17:09.444 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:09.444 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:17:09.444 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:09.445 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:09.445 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:09.445 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:09.445 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:17:09.446 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:09.446 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:09.446 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:17:09.447 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:09.447 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:09.447 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:09.447 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:17:09.448 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:09.448 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:09.448 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:09.448 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:09.449 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:17:09.449 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:09.449 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:09.449 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:17:09.450 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:09.450 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:09.450 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:09.450 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:17:09.451 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:09.451 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:09.451 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:09.452 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:09.452 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:17:09.452 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:09.452 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:09.453 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:17:09.453 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:09.453 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:09.453 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:09.454 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:17:09.454 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:09.454 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:09.455 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:09.455 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:17:09.455 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:09.456 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:17:09.456 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:17:09.456 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:17:09.456 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:09.457 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:17:09.458 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.762816_197.1 2021-11-11T21:17:14.459 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:14.460 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:14.461 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:17:14.503 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:17:14.504 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:17:14.577 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:17:14.577 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:17:14.577 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:17:14.578 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:17:14.578 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:17:14.578 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:17:14.579 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:14.579 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:14.579 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:17:14.579 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:14.579 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:17:14.580 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:14.580 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:14.580 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:17:14.580 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:14.581 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:17:14.581 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:14.581 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:17:14.581 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:14.582 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:14.582 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:14.582 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:14.582 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:17:14.583 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:14.583 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:14.583 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:17:14.583 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:14.584 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:14.584 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:14.584 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:17:14.584 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:14.585 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:14.585 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:14.585 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:14.585 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:17:14.586 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:14.586 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:14.586 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:17:14.586 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:14.586 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:14.587 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:14.587 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:17:14.587 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:14.587 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:14.588 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:14.588 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:14.588 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:17:14.588 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:14.589 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:14.589 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:17:14.589 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:14.589 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:14.589 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:14.590 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:17:14.590 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:14.590 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:14.590 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:14.591 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:17:14.591 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:14.591 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:17:14.591 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:17:14.591 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:17:14.592 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:14.592 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:17:14.593 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.762816_197.1 2021-11-11T21:17:19.594 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:19.595 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:19.596 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:17:19.639 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:17:19.640 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:17:19.708 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:17:19.708 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:17:19.709 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:17:19.709 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:17:19.709 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:17:19.710 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:17:19.710 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:19.710 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:19.710 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:17:19.711 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:19.711 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:17:19.711 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:19.711 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:19.712 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:17:19.712 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:19.713 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:17:19.713 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:19.713 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:17:19.714 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:19.714 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:19.714 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:19.715 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:19.715 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:17:19.716 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:19.716 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:19.716 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:17:19.717 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:19.717 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:19.717 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:19.718 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:17:19.718 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:19.719 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:19.719 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:19.719 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:19.720 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:17:19.720 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:19.721 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:19.721 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:17:19.721 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:19.722 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:19.722 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:19.723 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:17:19.723 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:19.723 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:19.724 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:19.724 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:19.725 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:17:19.725 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:19.725 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:19.725 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:17:19.726 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:19.726 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:19.726 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:19.726 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:17:19.726 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:19.727 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:19.727 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:19.727 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:17:19.727 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:19.728 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:17:19.728 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:17:19.728 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:17:19.728 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:19.728 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:17:19.730 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.762816_197.1 2021-11-11T21:17:24.732 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:24.732 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:24.733 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:17:24.777 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:17:24.777 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:17:24.843 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:17:24.844 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:17:24.844 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:17:24.844 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:17:24.844 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:17:24.845 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:17:24.845 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:24.845 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:24.845 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:17:24.845 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:24.846 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:17:24.846 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:24.846 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:24.846 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:17:24.847 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:24.847 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:17:24.847 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:24.847 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:17:24.848 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:24.848 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:24.851 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:24.851 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:24.852 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:17:24.852 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:24.852 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:24.852 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:17:24.853 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:24.853 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:24.853 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:24.853 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:17:24.854 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:24.854 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:24.854 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:24.854 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:24.855 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:17:24.855 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:24.855 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:24.855 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:17:24.855 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:24.856 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:24.856 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:24.856 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:17:24.857 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:24.857 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:24.857 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:24.857 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:24.857 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:17:24.858 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:24.858 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:24.858 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:17:24.858 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:24.859 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:24.859 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:24.859 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:17:24.859 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:24.860 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:24.860 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:24.860 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:17:24.860 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:24.861 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:17:24.861 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:17:24.861 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:17:24.861 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:24.862 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:17:24.863 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.825641_203.1 2021-11-11T21:17:29.865 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:29.866 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:29.866 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:17:29.909 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:17:29.909 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:17:29.979 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:17:29.979 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:17:29.979 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:17:29.979 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:17:29.980 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:17:29.980 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:17:29.980 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:29.980 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:29.981 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:17:29.981 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:29.981 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:17:29.981 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:29.982 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:29.982 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:17:29.982 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:29.982 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:17:29.983 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:29.983 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:17:29.983 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:29.983 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:29.983 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:29.984 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:29.984 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:17:29.984 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:29.984 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:29.985 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:17:29.985 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:29.985 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:29.985 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:29.985 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:17:29.986 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:29.986 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:29.986 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:29.987 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:29.987 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:17:29.987 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:29.988 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:29.988 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:17:29.988 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:29.988 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:29.988 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:29.989 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:17:29.989 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:29.989 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:29.989 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:29.990 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:29.990 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:17:29.990 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:29.990 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:29.991 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:17:29.991 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:29.991 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:29.991 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:29.991 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:17:29.992 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:29.992 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:29.992 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:29.992 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:17:29.993 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:29.993 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:17:29.993 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:17:29.993 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:17:29.994 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:29.994 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:17:29.995 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.831934_205.1 2021-11-11T21:17:34.996 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:34.997 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:34.998 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:17:35.040 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:17:35.041 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:17:35.118 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:17:35.119 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:17:35.119 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:17:35.119 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:17:35.119 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:17:35.120 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:17:35.120 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:35.120 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:35.120 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:17:35.120 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:35.121 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:17:35.121 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:35.121 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:35.121 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:17:35.122 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:35.122 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:17:35.122 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:35.122 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:17:35.123 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:35.123 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:35.123 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:35.123 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:35.123 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:17:35.124 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:35.124 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:35.124 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:17:35.124 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:35.125 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:35.125 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:35.125 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:17:35.125 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:35.125 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:35.126 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:35.126 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:35.126 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:17:35.126 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:35.126 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:35.127 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:17:35.127 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:35.127 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:35.127 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:35.128 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:17:35.128 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:35.128 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:35.128 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:35.128 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:35.129 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:17:35.129 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:35.129 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:35.129 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:17:35.130 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:35.130 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:35.130 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:35.130 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:17:35.131 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:35.131 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:35.132 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:35.132 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:17:35.132 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:35.132 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:17:35.133 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:17:35.133 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:17:35.133 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:35.133 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:17:35.135 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:17:40.136 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:40.137 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:40.137 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:17:40.180 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:17:40.180 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:17:40.243 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:17:40.244 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:17:40.244 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:17:40.244 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:17:40.244 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:17:40.245 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:17:40.245 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:40.245 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:40.245 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:17:40.246 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:40.246 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:17:40.246 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:40.246 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:40.247 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:17:40.247 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:40.247 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:17:40.247 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:40.248 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:17:40.248 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:40.248 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:40.249 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:40.249 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:40.249 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:17:40.249 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:40.250 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:40.250 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:17:40.250 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:40.250 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:40.251 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:40.251 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:17:40.251 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:40.251 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:40.252 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:40.252 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:40.252 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:17:40.252 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:40.252 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:40.253 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:17:40.253 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:40.253 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:40.253 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:40.254 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:17:40.254 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:40.254 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:40.255 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:40.255 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:40.255 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:17:40.255 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:40.256 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:40.256 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:17:40.256 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:40.256 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:40.256 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:40.257 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:17:40.257 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:40.257 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:40.257 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:40.258 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:17:40.258 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:40.258 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:17:40.258 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:17:40.258 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:17:40.259 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:40.259 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:17:40.260 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:17:45.261 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:45.262 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:45.263 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:17:45.304 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:17:45.305 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:17:45.370 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:17:45.371 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:17:45.371 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:17:45.371 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:17:45.371 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:17:45.372 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:17:45.372 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:45.372 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:45.372 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:17:45.373 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:45.373 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:17:45.373 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:45.373 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:45.374 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:17:45.374 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:45.374 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:17:45.374 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:45.375 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:17:45.375 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:45.375 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:45.375 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:45.376 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:45.376 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:17:45.376 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:45.376 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:45.377 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:17:45.377 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:45.377 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:45.377 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:45.377 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:17:45.378 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:45.378 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:45.378 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:45.378 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:45.378 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:17:45.379 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:45.379 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:45.379 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:17:45.379 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:45.380 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:45.380 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:45.380 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:17:45.380 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:45.380 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:45.381 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:45.381 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:45.381 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:17:45.381 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:45.382 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:45.382 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:17:45.382 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:45.382 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:45.383 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:45.383 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:17:45.383 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:45.383 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:45.384 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:45.384 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:17:45.384 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:45.384 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:17:45.385 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:17:45.385 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:17:45.385 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:45.385 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:17:45.386 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:17:50.387 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:50.388 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:50.389 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:17:50.431 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:17:50.431 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:17:50.495 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:17:50.495 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:17:50.495 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:17:50.496 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:17:50.496 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:17:50.496 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:17:50.496 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:50.497 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:50.497 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:17:50.497 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:50.497 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:17:50.498 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:50.498 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:50.498 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:17:50.498 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:50.499 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:17:50.499 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:50.499 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:17:50.499 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:50.500 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:50.500 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:50.500 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:50.500 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:17:50.501 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:50.501 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:50.501 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:17:50.501 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:50.502 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:50.502 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:50.502 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:17:50.502 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:50.503 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:50.503 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:50.503 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:50.503 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:17:50.504 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:50.504 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:50.504 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.049364_216.1", 2021-11-11T21:17:50.504 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:50.505 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:50.505 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:50.505 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.049364Z", 2021-11-11T21:17:50.505 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:50.505 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:50.506 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:50.506 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:50.506 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:17:50.506 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:50.507 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:50.507 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:17:50.507 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:50.508 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:50.508 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:50.508 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:17:50.508 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:50.509 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:50.509 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:50.511 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:17:50.511 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:50.511 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:17:50.511 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:17:50.512 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:17:50.512 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:50.512 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:17:50.513 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665225.049364_216.1 2021-11-11T21:17:55.515 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:55.516 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:17:55.517 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:17:55.596 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:17:55.597 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:17:55.728 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:17:55.728 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:17:55.729 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:17:55.729 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:17:55.729 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:17:55.729 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:17:55.729 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:55.730 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:55.730 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:17:55.730 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:55.730 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:17:55.731 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:55.731 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:55.731 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:17:55.731 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:55.731 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:17:55.732 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:55.732 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:17:55.732 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:55.732 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:55.733 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:55.733 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:55.733 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:17:55.733 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:55.733 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:55.734 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:17:55.734 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:55.734 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:55.734 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:55.734 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:17:55.735 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:55.735 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:55.735 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:55.735 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:55.736 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:17:55.736 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:55.736 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:55.736 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:17:55.736 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:55.737 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:55.737 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:55.737 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:17:55.738 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:55.738 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:55.738 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:55.738 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:17:55.739 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:17:55.740 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:17:55.740 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:17:55.740 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:17:55.740 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:17:55.741 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:17:55.741 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:17:55.741 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:17:55.741 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:17:55.741 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:55.742 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:55.742 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:17:55.742 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:17:55.742 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:17:55.743 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:17:55.743 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:17:55.743 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:17:55.743 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:17:55.745 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:18:00.746 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:00.747 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:00.748 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:18:00.817 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:18:00.817 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:18:01.062 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:18:01.062 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:18:01.062 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:18:01.063 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:18:01.063 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:18:01.063 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:18:01.063 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:01.064 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:01.064 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:18:01.064 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:01.064 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:18:01.064 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:01.065 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:01.065 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:18:01.065 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:01.065 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:18:01.066 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:01.066 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:18:01.066 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:01.066 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:01.067 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:01.067 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:01.067 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:18:01.067 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:01.068 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:01.068 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:18:01.068 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:01.068 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:01.068 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:01.069 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:18:01.069 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:01.069 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:01.069 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:01.070 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:01.070 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:18:01.070 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:01.070 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:01.070 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:18:01.071 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:01.071 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:01.071 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:01.071 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:18:01.071 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:01.072 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:01.072 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:01.072 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:01.072 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:18:01.073 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:01.073 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:01.073 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:18:01.073 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:01.073 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:01.074 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:01.074 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:18:01.074 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:01.074 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:01.074 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:01.075 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:18:01.075 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:01.075 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:18:01.075 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:18:01.076 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:18:01.076 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:01.076 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:18:01.077 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.831934_205.1 2021-11-11T21:18:06.079 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:06.079 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:06.080 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:18:06.123 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:18:06.123 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:18:06.201 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:18:06.202 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:18:06.202 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:18:06.202 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:18:06.203 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:18:06.203 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:18:06.203 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:06.203 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:06.204 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:18:06.204 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:06.204 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:18:06.204 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:06.204 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:06.205 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:18:06.205 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:06.205 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:18:06.205 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:06.206 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:18:06.206 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:06.206 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:06.206 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:06.207 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:06.207 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:18:06.207 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:06.207 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:06.208 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:18:06.208 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:06.208 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:06.208 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:06.209 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:18:06.209 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:06.209 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:06.210 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:06.210 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:06.210 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:18:06.210 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:06.211 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:06.211 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:18:06.211 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:06.211 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:06.211 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:06.212 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:18:06.212 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:06.212 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:06.212 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:06.213 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:06.213 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:18:06.213 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:06.213 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:06.213 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:18:06.214 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:06.214 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:06.214 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:06.214 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:18:06.215 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:06.215 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:06.215 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:06.215 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:18:06.216 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:06.216 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:18:06.216 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:18:06.216 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:18:06.216 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:06.217 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:18:06.218 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.831934_205.1 2021-11-11T21:18:11.219 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:11.220 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:11.221 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:18:11.264 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:18:11.264 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:18:11.332 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:18:11.332 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:18:11.333 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:18:11.333 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:18:11.333 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:18:11.333 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:18:11.333 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:11.334 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:11.334 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:18:11.334 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:11.334 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:18:11.335 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:11.335 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:11.335 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:18:11.335 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:11.336 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:18:11.336 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:11.336 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:18:11.336 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:11.337 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:11.337 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:11.337 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:11.337 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:18:11.337 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:11.338 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:11.338 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:18:11.338 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:11.338 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:11.339 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:11.339 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:18:11.339 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:11.339 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:11.340 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:11.340 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:11.340 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:18:11.340 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:11.340 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:11.341 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:18:11.341 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:11.342 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:11.342 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:11.342 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:18:11.342 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:11.343 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:11.343 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:11.343 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:11.343 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:18:11.343 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:11.344 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:11.344 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:18:11.344 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:11.344 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:11.345 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:11.345 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:18:11.345 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:11.345 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:11.345 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:11.346 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:18:11.346 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:11.346 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:18:11.346 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:18:11.346 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:18:11.347 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:11.347 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:18:11.348 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.830110_204.1 2021-11-11T21:18:16.349 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:16.350 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:16.350 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:18:16.427 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:18:16.427 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:18:16.557 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:18:16.558 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:18:16.558 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:18:16.558 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:18:16.558 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:18:16.559 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:18:16.559 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:16.559 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:16.559 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:18:16.560 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:16.560 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:18:16.560 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:16.560 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:16.560 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:18:16.561 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:16.561 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:18:16.561 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:16.561 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:18:16.562 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:16.562 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:16.562 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:16.562 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:16.562 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:18:16.563 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:16.563 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:16.563 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:18:16.563 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:16.564 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:16.564 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:16.564 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:18:16.564 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:16.565 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:16.565 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:16.565 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:16.565 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:18:16.566 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:16.566 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:16.566 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:18:16.566 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:16.566 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:16.567 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:16.567 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:18:16.567 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:16.567 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:16.568 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:16.568 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:16.568 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:18:16.568 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:16.569 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:16.569 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:18:16.569 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:16.569 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:16.570 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:16.570 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:18:16.570 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:16.570 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:16.570 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:16.571 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:18:16.571 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:16.571 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:18:16.571 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:18:16.572 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:18:16.572 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:16.572 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:18:16.573 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.825641_203.1 2021-11-11T21:18:21.574 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:21.575 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:21.576 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:18:21.644 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:18:21.644 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:18:21.720 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:18:21.720 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:18:21.721 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:18:21.721 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:18:21.721 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:18:21.721 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:18:21.721 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:21.722 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:21.722 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:18:21.722 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:21.723 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:18:21.723 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:21.723 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:21.723 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:18:21.723 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:21.724 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:18:21.724 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:21.724 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:18:21.724 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:21.725 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:21.725 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:21.725 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:21.725 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:18:21.726 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:21.726 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:21.726 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:18:21.726 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:21.727 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:21.727 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:21.727 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:18:21.727 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:21.728 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:21.728 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:21.728 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:21.728 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:18:21.729 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:21.729 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:21.729 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:18:21.729 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:21.730 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:21.730 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:21.730 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:18:21.730 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:21.731 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:21.731 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:21.731 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:21.731 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:18:21.731 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:21.732 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:21.732 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:18:21.732 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:21.732 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:21.733 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:21.733 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:18:21.733 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:21.733 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:21.733 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:21.734 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:18:21.734 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:21.734 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:18:21.734 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:18:21.735 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:18:21.735 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:21.735 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:18:21.737 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:18:26.737 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:26.738 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:26.739 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:18:26.809 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:18:26.809 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:18:26.902 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:18:26.902 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:18:26.902 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:18:26.903 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:18:26.903 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:18:26.903 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:18:26.903 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:26.904 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:26.904 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:18:26.904 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:26.904 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:18:26.904 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:26.905 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:26.905 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:18:26.905 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:26.905 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:18:26.906 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:26.906 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:18:26.906 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:26.906 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:26.906 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:26.907 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:26.907 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:18:26.907 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:26.907 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:26.908 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:18:26.908 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:26.908 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:26.908 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:26.908 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:18:26.909 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:26.909 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:26.909 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:26.910 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:26.910 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:18:26.910 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:26.910 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:26.911 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:18:26.911 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:26.911 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:26.911 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:26.912 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:18:26.912 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:26.912 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:26.912 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:26.913 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:26.913 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:18:26.913 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:26.913 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:26.913 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:18:26.914 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:26.914 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:26.914 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:26.914 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:18:26.915 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:26.915 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:26.915 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:26.915 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:18:26.915 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:26.916 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:18:26.916 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:18:26.916 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:18:26.916 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:26.917 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:18:26.918 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.783136_199.1 2021-11-11T21:18:31.920 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:31.921 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:31.921 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:18:31.993 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:18:31.993 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:18:32.168 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:18:32.168 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:18:32.168 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:18:32.169 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:18:32.169 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:18:32.169 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:18:32.169 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:32.169 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:32.170 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:18:32.170 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:32.170 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:18:32.170 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:32.171 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:32.171 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:18:32.171 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:32.171 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:18:32.172 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:32.172 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:18:32.172 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:32.172 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:32.172 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:32.173 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:32.173 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:18:32.173 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:32.173 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:32.174 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:18:32.174 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:32.174 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:32.174 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:32.175 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:18:32.175 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:32.175 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:32.175 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:32.176 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:32.176 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:18:32.176 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:32.176 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:32.176 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:18:32.177 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:32.177 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:32.177 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:32.177 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:18:32.178 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:32.178 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:32.178 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:32.178 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:32.179 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:18:32.179 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:32.179 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:32.179 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:18:32.180 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:32.180 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:32.180 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:32.180 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:18:32.180 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:32.181 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:32.181 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:32.181 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:18:32.181 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:32.182 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:18:32.182 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:18:32.182 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:18:32.182 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:32.183 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:18:32.184 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:18:37.186 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:37.187 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:37.187 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:18:37.409 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:18:37.409 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:18:37.515 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:18:37.515 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:18:37.516 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:18:37.516 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:18:37.516 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:18:37.516 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:18:37.516 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:37.517 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:37.517 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:18:37.517 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:37.517 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:18:37.518 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:37.518 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:37.518 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:18:37.518 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:37.519 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:18:37.519 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:37.519 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:18:37.519 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:37.519 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:37.520 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:37.520 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:37.520 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:18:37.520 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:37.521 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:37.521 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:18:37.521 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:37.521 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:37.521 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:37.522 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:18:37.522 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:37.522 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:37.522 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:37.523 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:37.523 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:18:37.523 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:37.523 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:37.523 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:18:37.524 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:37.524 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:37.524 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:37.524 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:18:37.524 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:37.525 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:37.525 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:37.525 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:37.525 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:18:37.526 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:37.526 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:37.526 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:18:37.526 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:37.526 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:37.527 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:37.527 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:18:37.527 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:37.527 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:37.527 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:37.528 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:18:37.528 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:37.528 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:18:37.528 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:18:37.529 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:18:37.529 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:37.529 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:18:37.530 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.789745_201.1 2021-11-11T21:18:42.531 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:42.532 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:42.533 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:18:42.740 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:18:42.740 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:18:42.865 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:18:42.865 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:18:42.865 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:18:42.866 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:18:42.866 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:18:42.866 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:18:42.866 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:42.867 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:42.867 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:18:42.867 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:42.867 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:18:42.867 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:42.868 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:42.868 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:18:42.868 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:42.868 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:18:42.869 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:42.869 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:18:42.869 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:42.869 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:42.870 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:42.870 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:42.870 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:18:42.871 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:42.871 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:42.872 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:18:42.872 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:42.872 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:42.873 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:42.873 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:18:42.873 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:42.873 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:42.874 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:42.874 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:42.874 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:18:42.874 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:42.875 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:42.875 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.050883_217.1", 2021-11-11T21:18:42.875 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:42.875 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:42.875 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:42.876 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.050883Z", 2021-11-11T21:18:42.876 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:42.876 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:42.876 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:42.877 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:42.877 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:18:42.877 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:42.877 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:42.877 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:18:42.878 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:42.878 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:42.878 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:42.878 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:18:42.879 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:42.879 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:42.879 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:42.879 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:18:42.879 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:42.880 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:18:42.880 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:18:42.880 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:18:42.880 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:42.880 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:18:42.882 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665225.050883_217.1 2021-11-11T21:18:47.884 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:47.884 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:47.885 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:18:48.030 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:18:48.030 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:18:48.097 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:18:48.098 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:18:48.098 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:18:48.098 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:18:48.099 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:18:48.099 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:18:48.099 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:48.099 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:48.099 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:18:48.100 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:48.100 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:18:48.100 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:48.100 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:48.101 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:18:48.101 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:48.101 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:18:48.101 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:48.101 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:18:48.102 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:48.102 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:48.102 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:48.102 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:48.103 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:18:48.103 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:48.103 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:48.103 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:18:48.104 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:48.104 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:48.104 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:48.104 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:18:48.104 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:48.105 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:48.105 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:48.105 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:48.105 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:18:48.105 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:48.106 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:48.106 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:18:48.106 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:48.107 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:48.107 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:48.107 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:18:48.107 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:48.108 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:48.108 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:48.108 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:48.108 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:18:48.109 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:48.109 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:48.109 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:18:48.109 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:48.109 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:48.110 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:48.110 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:18:48.110 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:48.110 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:48.111 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:48.111 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:18:48.111 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:48.111 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:18:48.111 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:18:48.112 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:18:48.112 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:48.112 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:18:48.113 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.762816_197.1 2021-11-11T21:18:53.114 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:53.115 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:53.116 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:18:53.157 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:18:53.158 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:18:53.232 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:18:53.232 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:18:53.232 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:18:53.233 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:18:53.233 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:18:53.233 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:18:53.233 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:53.234 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:53.234 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:18:53.234 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:53.234 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:18:53.235 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:53.235 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:53.235 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:18:53.235 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:53.235 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:18:53.236 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:53.236 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:18:53.236 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:53.236 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:53.237 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:53.237 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:53.237 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:18:53.237 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:53.238 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:53.238 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:18:53.238 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:53.238 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:53.238 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:53.239 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:18:53.239 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:53.239 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:53.239 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:53.240 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:53.240 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:18:53.240 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:53.240 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:53.241 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:18:53.241 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:53.241 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:53.241 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:53.241 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:18:53.242 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:53.242 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:53.242 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:53.242 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:18:53.243 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:18:53.243 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:18:53.243 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:18:53.243 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:18:53.244 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:18:53.244 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:18:53.244 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:18:53.244 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:18:53.244 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:18:53.245 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:53.245 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:53.245 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:18:53.245 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:18:53.246 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:18:53.246 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:18:53.246 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:18:53.246 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:18:53.247 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:18:53.248 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665227.956742_262.1 target=1_1636665224.830110_204.1 2021-11-11T21:18:58.250 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_delete_notempty ... FAIL 2021-11-11T21:18:58.251 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-39 2021-11-11T21:18:58.287 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-40 2021-11-11T21:18:59.378 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-41 2021-11-11T21:18:59.452 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:59.453 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:18:59.453 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:00.214 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:00.214 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:00.269 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:19:00.292 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:19:00.292 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:19:00.292 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:00.292 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:19:00.293 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:19:00.293 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:19:00.293 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:19:00.293 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:19:00.294 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:19:00.294 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:19:00.294 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665539.417369_286.1", 2021-11-11T21:19:00.295 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:18:59.417369Z" 2021-11-11T21:19:00.295 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:19:00.295 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:19:00.295 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:00.296 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:19:00.296 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:19:00.296 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:19:00.297 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:19:00.298 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:00.299 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:00.299 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:00.381 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:00.381 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:00.516 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:19:00.516 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:19:00.516 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:19:00.517 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:19:00.517 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:19:00.517 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:19:00.518 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:00.518 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:00.518 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:19:00.518 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:00.519 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:19:00.519 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:00.519 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:00.519 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:00.520 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:00.520 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:19:00.520 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:00.520 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:19:00.521 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:00.521 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:00.521 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:00.521 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:00.522 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:19:00.522 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:00.522 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:00.522 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:19:00.523 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:00.523 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:00.523 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:00.523 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:19:00.523 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:00.524 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:00.524 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:00.524 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:00.524 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:19:00.525 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:00.525 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:00.525 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:19:00.525 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:00.526 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:00.526 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:00.526 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:19:00.526 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:00.526 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:00.527 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:00.527 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:00.527 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:19:00.527 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:00.528 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:00.528 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:00.528 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:00.528 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:00.528 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:00.529 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:19:00.529 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:00.529 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:00.529 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:00.530 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:19:00.530 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:00.530 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:19:00.530 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:19:00.531 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:19:00.531 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:00.531 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:19:00.532 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:19:05.533 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:05.534 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:05.535 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:05.620 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:05.620 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:05.885 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:19:05.886 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:19:05.887 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:19:05.887 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:19:05.887 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:19:05.888 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:19:05.888 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:05.889 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:05.889 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:19:05.889 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:05.890 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:19:05.890 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:05.891 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:05.891 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:05.892 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:05.892 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:19:05.893 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:05.893 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:19:05.894 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:05.894 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:05.895 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:05.895 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:05.896 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:19:05.896 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:05.897 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:05.897 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:19:05.897 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:05.898 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:05.898 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:05.899 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:19:05.899 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:05.900 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:05.900 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:05.901 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:05.901 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:19:05.902 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:05.902 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:05.903 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:19:05.903 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:05.904 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:05.904 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:05.904 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:19:05.905 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:05.905 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:05.906 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:05.906 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:05.906 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:19:05.907 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:05.907 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:05.907 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:05.908 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:05.908 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:05.908 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:05.909 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:19:05.909 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:05.909 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:05.909 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:05.910 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:19:05.910 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:05.910 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:19:05.910 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:19:05.910 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:19:05.911 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:05.911 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:19:05.912 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.789745_201.1 2021-11-11T21:19:10.914 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:10.915 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:10.915 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:10.998 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:10.999 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:11.219 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:19:11.219 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:19:11.219 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:19:11.220 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:19:11.220 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:19:11.220 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:19:11.220 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:11.221 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:11.221 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:19:11.221 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:11.221 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:19:11.222 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:11.222 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:11.222 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:11.222 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:11.223 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:19:11.223 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:11.223 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:19:11.223 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:11.224 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:11.224 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:11.224 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:11.224 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:19:11.225 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:11.225 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:11.225 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:19:11.226 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:11.226 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:11.226 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:11.226 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:19:11.227 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:11.227 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:11.227 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:11.227 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:11.228 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:19:11.228 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:11.228 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:11.228 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:19:11.228 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:11.229 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:11.229 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:11.229 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:19:11.229 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:11.229 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:11.230 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:11.230 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:11.230 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:19:11.230 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:11.230 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:11.231 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:11.231 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:11.231 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:11.231 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:11.232 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:19:11.232 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:11.232 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:11.232 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:11.233 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:19:11.233 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:11.233 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:19:11.233 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:19:11.233 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:19:11.234 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:11.234 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:19:11.235 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:19:16.237 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:16.237 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:16.238 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:16.328 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:16.329 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:16.565 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:19:16.566 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:19:16.566 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:19:16.566 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:19:16.566 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:19:16.567 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:19:16.567 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:16.567 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:16.567 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:19:16.568 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:16.568 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:19:16.568 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:16.568 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:16.568 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:16.569 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:16.569 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:19:16.569 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:16.569 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:19:16.570 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:16.570 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:16.570 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:16.570 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:16.571 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:19:16.571 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:16.571 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:16.571 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:19:16.571 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:16.572 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:16.572 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:16.572 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:19:16.572 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:16.573 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:16.573 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:16.573 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:16.573 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:19:16.573 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:16.574 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:16.574 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:19:16.574 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:16.574 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:16.575 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:16.575 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:19:16.575 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:16.575 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:16.576 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:16.576 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:16.576 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:19:16.576 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:16.576 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:16.577 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:16.577 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:16.577 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:16.577 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:16.578 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:19:16.578 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:16.578 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:16.578 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:16.578 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:19:16.579 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:16.579 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:19:16.579 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:19:16.579 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:19:16.580 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:16.580 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:19:16.581 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.787834_200.1 2021-11-11T21:19:21.583 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:21.584 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:21.584 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:21.671 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:21.672 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:22.022 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:19:22.023 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:19:22.023 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:19:22.023 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:19:22.024 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:19:22.024 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:19:22.024 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:22.024 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:22.024 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:19:22.025 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:22.025 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:19:22.025 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:22.025 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:22.026 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:22.026 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:22.026 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:19:22.026 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:22.027 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:19:22.027 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:22.027 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:22.028 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:22.028 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:22.028 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:19:22.028 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:22.029 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:22.029 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:19:22.029 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:22.030 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:22.030 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:22.030 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:19:22.030 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:22.031 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:22.031 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:22.031 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:22.031 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:19:22.032 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:22.032 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:22.032 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:19:22.033 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:22.033 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:22.033 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:22.033 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:19:22.033 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:22.034 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:22.034 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:22.034 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:22.034 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:19:22.035 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:22.035 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:22.035 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:22.036 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:22.036 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:22.036 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:22.036 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:19:22.036 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:22.037 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:22.037 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:22.037 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:19:22.038 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:22.038 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:19:22.038 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:19:22.038 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:19:22.039 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:22.039 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:19:22.040 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:19:27.042 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:27.043 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:27.043 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:27.142 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:27.142 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:27.579 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:19:27.579 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:19:27.579 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:19:27.579 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:19:27.580 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:19:27.580 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:19:27.580 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:27.580 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:27.581 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:19:27.581 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:27.581 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:19:27.581 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:27.581 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:27.582 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:27.582 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:27.582 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:19:27.582 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:27.583 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:19:27.583 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:27.583 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:27.583 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:27.584 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:27.584 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:19:27.585 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:27.585 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:27.585 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:19:27.585 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:27.585 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:27.586 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:27.586 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:19:27.586 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:27.586 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:27.587 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:27.587 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:27.587 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:19:27.587 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:27.588 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:27.588 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:19:27.588 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:27.588 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:27.589 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:27.589 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:19:27.589 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:27.589 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:27.590 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:27.590 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:27.590 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:19:27.590 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:27.590 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:27.591 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:27.591 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:27.591 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:27.591 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:27.592 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:19:27.592 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:27.592 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:27.592 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:27.592 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:19:27.593 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:27.593 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:19:27.593 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:19:27.593 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:19:27.594 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:27.594 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:19:27.595 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:19:32.597 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:32.598 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:32.598 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:32.707 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:32.707 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:32.920 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:19:32.921 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:19:32.921 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:19:32.921 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:19:32.921 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:19:32.921 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:19:32.922 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:32.922 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:32.922 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:19:32.922 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:32.923 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:19:32.923 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:32.923 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:32.923 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:32.924 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:32.924 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:19:32.924 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:32.924 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:19:32.925 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:32.925 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:32.925 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:32.925 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:32.925 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:19:32.926 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:32.926 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:32.926 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:19:32.926 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:32.927 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:32.927 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:32.927 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:19:32.927 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:32.927 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:32.928 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:32.928 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:32.928 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:19:32.928 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:32.929 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:32.929 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.838825_206.1", 2021-11-11T21:19:32.929 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:32.929 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:32.929 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:32.930 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.838825Z", 2021-11-11T21:19:32.930 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:32.930 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:32.930 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:32.931 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:32.931 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:19:32.931 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:32.931 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:32.931 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:32.932 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:32.932 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:32.932 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:32.932 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:19:32.933 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:32.933 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:32.933 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:32.933 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:19:32.933 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:32.934 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:19:32.934 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:19:32.934 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:19:32.934 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:32.935 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:19:32.936 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.838825_206.1 2021-11-11T21:19:37.941 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:37.941 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:37.942 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:38.024 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:38.024 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:38.596 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:19:38.597 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:19:38.597 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:19:38.597 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:19:38.598 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:19:38.598 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:19:38.598 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:38.598 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:38.599 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:19:38.599 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:38.599 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:19:38.599 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:38.600 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:38.600 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:38.600 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:38.600 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:19:38.601 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:38.601 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:19:38.601 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:38.601 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:38.601 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:38.602 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:38.602 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:19:38.602 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:38.602 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:38.603 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:19:38.603 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:38.603 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:38.603 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:38.604 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:19:38.604 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:38.604 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:38.604 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:38.605 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:38.605 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:19:38.605 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:38.605 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:38.605 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:19:38.606 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:38.606 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:38.606 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:38.606 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:19:38.607 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:38.607 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:38.607 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:38.607 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:38.608 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:19:38.608 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:38.608 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:38.608 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:38.608 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:38.609 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:38.609 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:38.609 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:19:38.609 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:38.610 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:38.610 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:38.610 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:19:38.610 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:38.611 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:19:38.611 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:19:38.611 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:19:38.611 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:38.611 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:19:38.613 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.768842_198.1 2021-11-11T21:19:43.614 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:43.615 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:43.615 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:43.862 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:43.863 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:44.170 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:19:44.171 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:19:44.171 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:19:44.171 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:19:44.172 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:19:44.172 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:19:44.172 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:44.173 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:44.173 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:19:44.173 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:44.173 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:19:44.174 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:44.174 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:44.174 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:44.174 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:44.175 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:19:44.175 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:44.175 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:19:44.175 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:44.176 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:44.176 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:44.176 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:44.176 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:19:44.177 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:44.177 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:44.177 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:19:44.177 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:44.178 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:44.178 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:44.178 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:19:44.178 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:44.179 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:44.179 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:44.179 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:44.179 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:19:44.180 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:44.180 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:44.180 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:19:44.180 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:44.181 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:44.181 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:44.181 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:19:44.181 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:44.182 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:44.182 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:44.182 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:44.183 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:19:44.183 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:44.183 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:44.183 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:44.184 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:44.184 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:44.184 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:44.184 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:19:44.185 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:44.185 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:44.185 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:44.185 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:19:44.186 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:44.186 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:19:44.186 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:19:44.186 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:19:44.187 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:44.187 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:19:44.188 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.783136_199.1 2021-11-11T21:19:49.190 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:49.191 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:49.191 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:49.238 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:49.238 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:49.611 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:19:49.611 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:19:49.612 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:19:49.612 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:19:49.612 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:19:49.612 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:19:49.612 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:49.613 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:49.613 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:19:49.613 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:49.613 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:19:49.614 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:49.614 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:49.614 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:49.614 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:49.615 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:19:49.615 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:49.615 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:19:49.615 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:49.616 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:49.616 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:49.616 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:49.616 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:19:49.617 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:49.617 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:49.617 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:19:49.617 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:49.617 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:49.618 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:49.618 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:19:49.618 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:49.618 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:49.619 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:49.619 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:49.619 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:19:49.619 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:49.619 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:49.620 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:19:49.620 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:49.620 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:49.620 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:49.621 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:19:49.621 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:49.621 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:49.621 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:49.621 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:49.622 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:19:49.622 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:49.622 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:49.622 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:49.623 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:49.623 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:49.623 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:49.623 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:19:49.624 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:49.624 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:49.624 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:49.624 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:19:49.624 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:49.625 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:19:49.625 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:19:49.625 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:19:49.625 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:49.626 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:19:49.627 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:19:54.628 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:54.628 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:19:54.629 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:19:54.917 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:19:54.917 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:19:55.205 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:19:55.206 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:19:55.206 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:19:55.206 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:19:55.207 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:19:55.207 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:19:55.207 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:55.207 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:55.208 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:19:55.208 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:55.208 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:19:55.208 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:55.208 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:55.209 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:19:55.209 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:55.209 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:19:55.209 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:55.210 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:19:55.210 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:55.210 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:55.210 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:55.210 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:55.211 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:19:55.211 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:55.211 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:55.211 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:19:55.212 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:55.212 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:55.212 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:55.212 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:19:55.212 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:55.213 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:55.213 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:55.213 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:55.213 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:19:55.214 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:55.214 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:55.214 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:19:55.214 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:55.214 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:55.215 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:55.215 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:19:55.215 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:55.215 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:55.215 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:55.216 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:19:55.216 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:19:55.216 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:19:55.216 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:19:55.217 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:19:55.217 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:19:55.217 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:19:55.217 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:19:55.217 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:19:55.218 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:19:55.218 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:55.218 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:55.219 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:19:55.219 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:19:55.219 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:19:55.219 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:19:55.220 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:19:55.220 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:19:55.220 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:19:55.222 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.761008_196.1 2021-11-11T21:20:00.223 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:00.224 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:00.224 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:20:00.309 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:20:00.309 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:20:00.689 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:20:00.690 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:20:00.690 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:20:00.690 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:20:00.690 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:20:00.691 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:20:00.691 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:00.691 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:00.692 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:20:00.692 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:00.692 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:20:00.692 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:00.693 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:00.693 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:20:00.693 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:00.694 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:20:00.694 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:00.694 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:20:00.694 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:00.695 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:00.695 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:00.695 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:00.695 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:20:00.696 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:00.696 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:00.696 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:20:00.697 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:00.697 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:00.697 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:00.698 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:20:00.698 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:00.698 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:00.698 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:00.699 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:00.699 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:20:00.699 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:00.699 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:00.700 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:20:00.700 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:00.700 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:00.700 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:00.700 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:20:00.701 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:00.701 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:00.701 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:00.701 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:00.702 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:20:00.702 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:00.702 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:00.702 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:20:00.703 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:00.703 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:00.703 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:00.703 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:20:00.703 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:00.704 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:00.704 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:00.704 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:20:00.704 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:00.705 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:20:00.705 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:20:00.705 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:20:00.705 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:00.705 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:20:00.707 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:20:05.708 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:05.709 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:05.710 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:20:05.795 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:20:05.795 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:20:06.054 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:20:06.054 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:20:06.054 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:20:06.055 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:20:06.055 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:20:06.055 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:20:06.055 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:06.056 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:06.056 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:20:06.056 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:06.056 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:20:06.057 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:06.057 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:06.057 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:20:06.057 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:06.057 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:20:06.058 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:06.058 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:20:06.058 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:06.059 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:06.059 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:06.059 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:06.059 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:20:06.060 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:06.060 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:06.060 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:20:06.060 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:06.060 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:06.061 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:06.061 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:20:06.061 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:06.061 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:06.062 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:06.062 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:06.062 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:20:06.062 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:06.062 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:06.063 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:20:06.063 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:06.063 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:06.063 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:06.064 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:20:06.064 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:06.064 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:06.064 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:06.064 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:06.065 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:20:06.065 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:06.065 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:06.065 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:20:06.066 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:06.066 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:06.066 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:06.066 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:20:06.066 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:06.067 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:06.067 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:06.067 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:20:06.067 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:06.068 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:20:06.068 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:20:06.068 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:20:06.068 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:06.068 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:20:06.069 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.783136_199.1 2021-11-11T21:20:11.071 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:11.072 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:11.072 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:20:11.144 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:20:11.144 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:20:11.481 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:20:11.481 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:20:11.481 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:20:11.482 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:20:11.482 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:20:11.482 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:20:11.482 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:11.483 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:11.483 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:20:11.483 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:11.483 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:20:11.484 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:11.484 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:11.484 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:20:11.484 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:11.484 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:20:11.485 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:11.485 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:20:11.485 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:11.485 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:11.486 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:11.486 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:11.486 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:20:11.486 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:11.486 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:11.487 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:20:11.487 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:11.487 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:11.487 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:11.488 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:20:11.488 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:11.488 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:11.488 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:11.489 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:11.489 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:20:11.489 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:11.489 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:11.489 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:20:11.490 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:11.490 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:11.490 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:11.490 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:20:11.491 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:11.491 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:11.491 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:11.491 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:11.492 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:20:11.492 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:11.492 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:11.492 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:20:11.493 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:11.493 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:11.493 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:11.493 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:20:11.494 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:11.494 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:11.494 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:11.494 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:20:11.495 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:11.495 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:20:11.495 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:20:11.495 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:20:11.496 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:11.496 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:20:11.497 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:20:16.498 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:16.499 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:16.500 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:20:16.605 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:20:16.606 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:20:17.024 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:20:17.025 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:20:17.025 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:20:17.025 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:20:17.025 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:20:17.025 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:20:17.026 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:17.026 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:17.026 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:20:17.026 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:17.027 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:20:17.027 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:17.027 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:17.027 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:20:17.028 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:17.028 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:20:17.028 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:17.028 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:20:17.028 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:17.029 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:17.029 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:17.029 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:17.029 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:20:17.030 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:17.030 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:17.030 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:20:17.030 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:17.031 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:17.031 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:17.031 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:20:17.031 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:17.031 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:17.032 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:17.032 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:17.032 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:20:17.032 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:17.033 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:17.033 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:20:17.033 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:17.033 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:17.033 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:17.034 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:20:17.034 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:17.034 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:17.034 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:17.035 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:17.035 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:20:17.035 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:17.035 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:17.036 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:20:17.036 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:17.036 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:17.036 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:17.037 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:20:17.037 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:17.037 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:17.037 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:17.037 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:20:17.038 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:17.038 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:20:17.038 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:20:17.038 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:20:17.039 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:17.039 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:20:17.040 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.789745_201.1 2021-11-11T21:20:22.041 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:22.042 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:22.043 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:20:22.129 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:20:22.130 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:20:22.480 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:20:22.481 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:20:22.481 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:20:22.481 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:20:22.482 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:20:22.482 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:20:22.482 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:22.482 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:22.482 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:20:22.483 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:22.483 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:20:22.483 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:22.483 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:22.484 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:20:22.484 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:22.484 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:20:22.484 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:22.485 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:20:22.485 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:22.485 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:22.485 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:22.485 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:22.486 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:20:22.486 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:22.486 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:22.486 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:20:22.487 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:22.487 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:22.487 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:22.487 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:20:22.487 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:22.488 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:22.488 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:22.488 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:22.488 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:20:22.489 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:22.489 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:22.489 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:20:22.489 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:22.490 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:22.490 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:22.490 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:20:22.490 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:22.491 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:22.491 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:22.491 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:22.491 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:20:22.492 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:22.492 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:22.492 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:20:22.492 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:22.493 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:22.493 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:22.493 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:20:22.493 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:22.494 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:22.494 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:22.494 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:20:22.494 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:22.494 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:20:22.495 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:20:22.495 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:20:22.495 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:22.495 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:20:22.497 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.761008_196.1 2021-11-11T21:20:27.498 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:27.500 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:27.500 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:20:27.581 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:20:27.581 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:20:27.839 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:20:27.840 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:20:27.840 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:20:27.840 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:20:27.840 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:20:27.841 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:20:27.841 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:27.841 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:27.841 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:20:27.842 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:27.842 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:20:27.842 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:27.842 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:27.842 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:20:27.843 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:27.843 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:20:27.843 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:27.843 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:20:27.844 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:27.844 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:27.844 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:27.844 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:27.844 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:20:27.845 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:27.845 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:27.845 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:20:27.845 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:27.845 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:27.846 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:27.846 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:20:27.846 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:27.846 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:27.847 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:27.847 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:27.847 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:20:27.847 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:27.848 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:27.848 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:20:27.848 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:27.848 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:27.848 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:27.849 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:20:27.849 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:27.849 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:27.849 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:27.850 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:27.850 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:20:27.850 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:27.850 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:27.850 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:20:27.851 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:27.851 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:27.851 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:27.851 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:20:27.851 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:27.852 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:27.852 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:27.852 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:20:27.852 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:27.852 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:20:27.853 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:20:27.853 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:20:27.853 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:27.853 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:20:27.855 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.768842_198.1 2021-11-11T21:20:32.856 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:32.857 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:32.857 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:20:32.955 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:20:32.955 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:20:33.292 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:20:33.292 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:20:33.293 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:20:33.293 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:20:33.293 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:20:33.293 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:20:33.293 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:33.294 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:33.294 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:20:33.294 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:33.294 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:20:33.295 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:33.295 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:33.295 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:20:33.295 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:33.296 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:20:33.296 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:33.296 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:20:33.296 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:33.296 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:33.297 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:33.297 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:33.297 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:20:33.297 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:33.298 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:33.298 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:20:33.298 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:33.298 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:33.299 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:33.299 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:20:33.299 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:33.299 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:33.299 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:33.300 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:33.300 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:20:33.300 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:33.300 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:33.301 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:20:33.301 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:33.301 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:33.301 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:33.301 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:20:33.302 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:33.302 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:33.302 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:33.302 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:33.303 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:20:33.303 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:33.303 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:33.303 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:20:33.303 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:33.304 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:33.304 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:33.304 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:20:33.304 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:33.305 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:33.305 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:33.305 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:20:33.305 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:33.306 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:20:33.306 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:20:33.306 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:20:33.306 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:33.306 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:20:33.308 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:20:38.309 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:38.310 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:38.311 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:20:38.416 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:20:38.417 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:20:38.824 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:20:38.825 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:20:38.825 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:20:38.825 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:20:38.826 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:20:38.826 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:20:38.826 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:38.826 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:38.827 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:20:38.827 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:38.827 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:20:38.827 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:38.828 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:38.828 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:20:38.828 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:38.828 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:20:38.829 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:38.829 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:20:38.829 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:38.829 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:38.830 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:38.830 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:38.830 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:20:38.830 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:38.831 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:38.831 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:20:38.831 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:38.831 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:38.832 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:38.832 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:20:38.832 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:38.832 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:38.833 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:38.833 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:38.833 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:20:38.833 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:38.834 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:38.834 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:20:38.834 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:38.834 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:38.835 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:38.835 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:20:38.835 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:38.835 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:38.835 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:38.836 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:38.836 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:20:38.836 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:38.836 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:38.837 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:20:38.837 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:38.837 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:38.837 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:38.838 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:20:38.838 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:38.838 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:38.838 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:38.839 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:20:38.839 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:38.839 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:20:38.840 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:20:38.840 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:20:38.840 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:38.840 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:20:38.841 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.831934_205.1 2021-11-11T21:20:43.842 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:43.842 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:43.843 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:20:43.931 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:20:43.931 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:20:44.336 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:20:44.337 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:20:44.337 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:20:44.337 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:20:44.337 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:20:44.338 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:20:44.338 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:44.338 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:44.339 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:20:44.339 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:44.339 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:20:44.339 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:44.340 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:44.340 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:20:44.340 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:44.340 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:20:44.341 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:44.341 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:20:44.341 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:44.341 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:44.341 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:44.342 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:44.342 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:20:44.342 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:44.342 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:44.343 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:20:44.343 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:44.343 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:44.343 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:44.343 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:20:44.344 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:44.344 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:44.344 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:44.344 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:44.344 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:20:44.345 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:44.345 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:44.345 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:20:44.345 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:44.345 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:44.346 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:44.346 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:20:44.346 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:44.346 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:44.347 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:44.347 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:44.347 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:20:44.347 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:44.348 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:44.348 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:20:44.348 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:44.348 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:44.348 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:44.349 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:20:44.349 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:44.349 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:44.349 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:44.350 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:20:44.350 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:44.350 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:20:44.350 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:20:44.351 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:20:44.351 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:44.351 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:20:44.352 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:20:49.353 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:49.354 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:49.355 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:20:49.436 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:20:49.437 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:20:50.008 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:20:50.009 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:20:50.009 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:20:50.009 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:20:50.009 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:20:50.010 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:20:50.010 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:50.010 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:50.010 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:20:50.011 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:50.011 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:20:50.011 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:50.011 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:50.011 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:20:50.012 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:50.012 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:20:50.013 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:50.013 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:20:50.013 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:50.013 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:50.014 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:50.014 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:50.014 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:20:50.014 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:50.014 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:50.015 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:20:50.015 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:50.015 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:50.015 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:50.016 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:20:50.016 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:50.016 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:50.016 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:50.017 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:50.017 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:20:50.017 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:50.017 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:50.017 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:20:50.018 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:50.018 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:50.018 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:50.018 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:20:50.019 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:50.019 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:50.019 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:50.019 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:50.020 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:20:50.020 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:50.020 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:50.020 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:20:50.021 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:50.021 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:50.021 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:50.021 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:20:50.022 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:50.022 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:50.022 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:50.023 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:20:50.023 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:50.023 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:20:50.023 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:20:50.024 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:20:50.024 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:50.024 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:20:50.025 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.831934_205.1 2021-11-11T21:20:55.026 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:55.027 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:20:55.028 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:20:55.104 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:20:55.104 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:20:55.359 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:20:55.360 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:20:55.360 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:20:55.360 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:20:55.361 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:20:55.361 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:20:55.361 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:55.361 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:55.361 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:20:55.362 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:55.362 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:20:55.362 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:55.363 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:55.363 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:20:55.363 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:55.363 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:20:55.364 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:55.364 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:20:55.364 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:55.364 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:55.364 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:55.365 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:55.365 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:20:55.365 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:55.365 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:55.366 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:20:55.366 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:55.366 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:55.366 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:55.367 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:20:55.367 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:55.367 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:55.368 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:55.368 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:55.368 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:20:55.368 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:55.368 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:55.369 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:20:55.369 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:55.369 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:55.369 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:55.369 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:20:55.370 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:55.370 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:55.370 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:55.370 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:20:55.371 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:20:55.371 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:20:55.371 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:20:55.371 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:20:55.371 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:20:55.372 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:20:55.372 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:20:55.372 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:20:55.372 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:20:55.372 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:55.373 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:55.373 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:20:55.373 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:20:55.373 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:20:55.374 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:20:55.374 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:20:55.374 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:20:55.374 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:20:55.375 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:21:00.377 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:00.378 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:00.379 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:21:00.592 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:21:00.592 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:21:01.131 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:21:01.132 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:21:01.132 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:21:01.132 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:21:01.133 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:21:01.133 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:21:01.133 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:01.133 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:01.134 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:21:01.134 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:01.134 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:21:01.134 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:01.134 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:01.135 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:21:01.135 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:01.135 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:21:01.136 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:01.136 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:21:01.136 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:01.137 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:01.137 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:01.137 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:01.138 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:21:01.138 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:01.138 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:01.139 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:21:01.139 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:01.140 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:01.140 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:01.140 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:21:01.141 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:01.141 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:01.142 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:01.142 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:01.142 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:21:01.143 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:01.143 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:01.143 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:21:01.144 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:01.144 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:01.144 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:01.144 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:21:01.145 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:01.145 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:01.145 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:01.145 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:01.145 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:21:01.146 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:01.146 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:01.146 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:21:01.146 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:01.147 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:01.147 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:01.147 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:21:01.147 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:01.147 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:01.148 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:01.148 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:21:01.148 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:01.148 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:21:01.149 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:21:01.149 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:21:01.149 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:01.149 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:21:01.151 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.789745_201.1 2021-11-11T21:21:06.152 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:06.153 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:06.154 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:21:06.866 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:21:06.866 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:21:06.980 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:21:06.981 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:21:06.981 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:21:06.981 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:21:06.981 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:21:06.982 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:21:06.982 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:06.982 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:06.982 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:21:06.983 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:06.983 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:21:06.983 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:06.983 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:06.984 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:21:06.984 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:06.984 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:21:06.984 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:06.984 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:21:06.985 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:06.985 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:06.985 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:06.985 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:06.986 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:21:06.986 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:06.986 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:06.986 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:21:06.987 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:06.987 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:06.987 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:06.987 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:21:06.988 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:06.988 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:06.988 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:06.988 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:06.988 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:21:06.989 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:06.989 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:06.989 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:21:06.989 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:06.990 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:06.990 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:06.990 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:21:06.990 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:06.991 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:06.991 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:06.991 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:06.991 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:21:06.992 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:06.992 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:06.992 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:21:06.992 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:06.992 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:06.993 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:06.993 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:21:06.993 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:06.993 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:06.994 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:06.994 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:21:06.994 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:06.994 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:21:06.995 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:21:06.995 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:21:06.995 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:06.995 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:21:06.996 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:21:11.998 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:11.999 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:12.000 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:21:12.082 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:21:12.082 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:21:12.335 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:21:12.336 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:21:12.336 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:21:12.336 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:21:12.337 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:21:12.337 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:21:12.337 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:12.337 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:12.338 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:21:12.338 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:12.338 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:21:12.338 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:12.338 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:12.339 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:21:12.339 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:12.339 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:21:12.339 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:12.340 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:21:12.340 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:12.340 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:12.340 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:12.340 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:12.341 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:21:12.341 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:12.341 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:12.341 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:21:12.342 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:12.342 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:12.342 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:12.342 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:21:12.342 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:12.343 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:12.343 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:12.343 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:12.344 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:21:12.344 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:12.344 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:12.344 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.863466_209.1", 2021-11-11T21:21:12.345 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:12.345 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:12.345 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:12.345 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.863466Z", 2021-11-11T21:21:12.346 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:12.346 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:12.347 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:12.347 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:12.347 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:21:12.347 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:12.348 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:12.348 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:21:12.348 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:12.349 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:12.349 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:12.349 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:21:12.349 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:12.350 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:12.350 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:12.350 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:21:12.350 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:12.351 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:21:12.351 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:21:12.351 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:21:12.352 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:12.352 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:21:12.353 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.863466_209.1 2021-11-11T21:21:17.354 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:17.355 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:17.356 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:21:17.442 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:21:17.442 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:21:17.665 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:21:17.666 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:21:17.666 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:21:17.666 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:21:17.667 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:21:17.667 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:21:17.667 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:17.667 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:17.668 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:21:17.668 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:17.668 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:21:17.668 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:17.668 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:17.669 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:21:17.669 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:17.669 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:21:17.669 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:17.670 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:21:17.670 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:17.670 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:17.670 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:17.671 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:17.671 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:21:17.671 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:17.671 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:17.672 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:21:17.672 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:17.672 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:17.672 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:17.673 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:21:17.673 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:17.673 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:17.673 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:17.674 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:17.674 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:21:17.674 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:17.674 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:17.675 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:21:17.675 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:17.675 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:17.675 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:17.675 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:21:17.676 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:17.676 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:17.676 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:17.676 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:17.677 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:21:17.677 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:17.677 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:17.677 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:21:17.678 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:17.678 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:17.678 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:17.678 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:21:17.679 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:17.679 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:17.679 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:17.679 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:21:17.680 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:17.680 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:21:17.680 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:21:17.680 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:21:17.680 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:17.681 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:21:17.682 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:21:22.684 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:22.685 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:22.685 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:21:22.777 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:21:22.777 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:21:23.135 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:21:23.135 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:21:23.136 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:21:23.136 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:21:23.136 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:21:23.136 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:21:23.137 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:23.137 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:23.137 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:21:23.137 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:23.138 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:21:23.138 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:23.138 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:23.138 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:21:23.139 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:23.139 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:21:23.139 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:23.139 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:21:23.140 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:23.140 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:23.140 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:23.140 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:23.141 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:21:23.141 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:23.141 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:23.142 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:21:23.142 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:23.142 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:23.142 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:23.143 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:21:23.143 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:23.143 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:23.143 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:23.143 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:23.144 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:21:23.144 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:23.144 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:23.144 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:21:23.145 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:23.145 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:23.145 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:23.145 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:21:23.145 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:23.146 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:23.146 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:23.146 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:23.146 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:21:23.147 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:23.147 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:23.147 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:21:23.147 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:23.148 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:23.148 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:23.148 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:21:23.148 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:23.149 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:23.149 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:23.149 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:21:23.149 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:23.150 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:21:23.150 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:21:23.150 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:21:23.150 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:23.151 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:21:23.152 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:21:28.154 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:28.154 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:28.155 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:21:28.217 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:21:28.218 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:21:28.636 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:21:28.636 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:21:28.637 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:21:28.637 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:21:28.637 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:21:28.638 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:21:28.638 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:28.638 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:28.638 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:21:28.639 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:28.639 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:21:28.639 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:28.639 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:28.639 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:21:28.640 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:28.640 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:21:28.640 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:28.640 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:21:28.641 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:28.641 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:28.641 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:28.641 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:28.642 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:21:28.642 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:28.642 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:28.642 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:21:28.643 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:28.643 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:28.643 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:28.644 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:21:28.644 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:28.644 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:28.644 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:28.645 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:28.645 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:21:28.645 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:28.645 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:28.646 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.050883_217.1", 2021-11-11T21:21:28.646 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:28.646 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:28.646 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:28.647 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.050883Z", 2021-11-11T21:21:28.647 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:28.647 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:28.647 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:28.648 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:28.648 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:21:28.648 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:28.648 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:28.649 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:21:28.649 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:28.649 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:28.649 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:28.650 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:21:28.650 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:28.650 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:28.650 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:28.651 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:21:28.651 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:28.651 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:21:28.651 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:21:28.651 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:21:28.652 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:28.652 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:21:28.652 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665225.050883_217.1 2021-11-11T21:21:33.654 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:33.655 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:33.655 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:21:33.707 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:21:33.708 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:21:33.782 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:21:33.783 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:21:33.783 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:21:33.783 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:21:33.783 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:21:33.784 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:21:33.784 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:33.784 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:33.784 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:21:33.785 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:33.785 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:21:33.785 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:33.785 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:33.786 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:21:33.786 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:33.786 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:21:33.786 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:33.787 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:21:33.787 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:33.787 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:33.787 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:33.787 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:33.788 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:21:33.788 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:33.788 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:33.788 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:21:33.789 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:33.789 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:33.789 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:33.789 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:21:33.789 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:33.790 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:33.790 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:33.790 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:33.790 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:21:33.791 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:33.791 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:33.791 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:21:33.792 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:33.792 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:33.792 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:33.792 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:21:33.793 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:33.793 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:33.793 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:33.793 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:33.793 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:21:33.794 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:33.794 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:33.794 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:21:33.794 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:33.795 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:33.795 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:33.795 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:21:33.795 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:33.796 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:33.796 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:33.796 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:21:33.796 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:33.796 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:21:33.797 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:21:33.797 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:21:33.797 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:33.797 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:21:33.799 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.789745_201.1 2021-11-11T21:21:38.800 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:38.801 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:38.801 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:21:38.897 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:21:38.898 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:21:39.253 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:21:39.254 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:21:39.254 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:21:39.254 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:21:39.254 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:21:39.255 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:21:39.255 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:39.255 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:39.255 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:21:39.256 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:39.256 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:21:39.256 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:39.256 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:39.257 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:21:39.257 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:39.257 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:21:39.257 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:39.258 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:21:39.258 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:39.258 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:39.258 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:39.259 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:39.259 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:21:39.259 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:39.259 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:39.259 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:21:39.260 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:39.261 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:39.261 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:39.262 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:21:39.262 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:39.262 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:39.262 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:39.263 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:39.263 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:21:39.263 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:39.263 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:39.264 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:21:39.264 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:39.264 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:39.264 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:39.265 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:21:39.265 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:39.265 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:39.265 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:39.266 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:39.266 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:21:39.266 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:39.266 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:39.266 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:21:39.267 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:39.267 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:39.267 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:39.267 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:21:39.268 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:39.268 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:39.268 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:39.268 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:21:39.269 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:39.269 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:21:39.269 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:21:39.269 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:21:39.269 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:39.270 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:21:39.272 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:21:44.272 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:44.273 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:44.274 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:21:44.359 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:21:44.359 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:21:44.915 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:21:44.916 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:21:44.916 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:21:44.916 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:21:44.917 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:21:44.917 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:21:44.917 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:44.917 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:44.917 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:21:44.918 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:44.918 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:21:44.918 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:44.918 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:44.919 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:21:44.919 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:44.919 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:21:44.919 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:44.919 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:21:44.920 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:44.920 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:44.920 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:44.920 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:44.921 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:21:44.921 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:44.921 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:44.921 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:21:44.921 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:44.922 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:44.922 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:44.922 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:21:44.922 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:44.923 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:44.923 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:44.923 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:44.923 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:21:44.923 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:44.924 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:44.924 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.914354_210.1", 2021-11-11T21:21:44.924 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:44.924 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:44.924 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:44.925 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.914354Z", 2021-11-11T21:21:44.925 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:44.925 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:44.925 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:44.926 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:44.926 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:21:44.926 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:44.926 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:44.926 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:21:44.927 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:44.927 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:44.927 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:44.927 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:21:44.928 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:44.928 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:44.928 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:44.928 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:21:44.928 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:44.929 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:21:44.929 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:21:44.929 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:21:44.929 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:44.929 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:21:44.931 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.914354_210.1 2021-11-11T21:21:47.460 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:21:47.459+0000 7f04ef001700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-5) 2021-11-11T21:21:49.932 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:49.933 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:49.933 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:21:50.043 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:21:50.043 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:21:50.576 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:21:50.577 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:21:50.577 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:21:50.577 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:21:50.578 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:21:50.578 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:21:50.578 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:50.578 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:50.579 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:21:50.579 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:50.579 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:21:50.579 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:50.579 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:50.580 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:21:50.580 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:50.580 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:21:50.580 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:50.581 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:21:50.581 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:50.581 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:50.581 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:50.582 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:50.582 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:21:50.582 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:50.582 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:50.583 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:21:50.583 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:50.583 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:50.583 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:50.583 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:21:50.584 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:50.584 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:50.584 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:50.584 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:50.585 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:21:50.585 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:50.585 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:50.585 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.977739_213.1", 2021-11-11T21:21:50.586 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:50.586 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:50.586 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:50.586 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.977739Z", 2021-11-11T21:21:50.587 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:50.587 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:50.587 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:50.587 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:50.588 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:21:50.588 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:50.588 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:50.588 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:21:50.589 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:50.589 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:50.589 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:50.589 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:21:50.589 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:50.590 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:50.590 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:50.590 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:21:50.590 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:50.591 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:21:50.591 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:21:50.591 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:21:50.591 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:50.592 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:21:50.593 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.977739_213.1 2021-11-11T21:21:55.594 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:55.595 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:21:55.596 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:21:55.818 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:21:55.818 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:21:56.136 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:21:56.136 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:21:56.136 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:21:56.137 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:21:56.137 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:21:56.137 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:21:56.137 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:56.137 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:56.138 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:21:56.138 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:56.138 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:21:56.138 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:56.139 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:56.139 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:21:56.139 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:56.139 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:21:56.140 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:56.140 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:21:56.140 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:56.140 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:56.140 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:56.141 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:56.141 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:21:56.141 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:56.141 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:56.142 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:21:56.142 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:56.142 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:56.142 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:56.143 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:21:56.143 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:56.143 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:56.143 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:56.143 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:56.144 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:21:56.144 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:56.144 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:56.144 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:21:56.145 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:56.145 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:56.145 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:56.145 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:21:56.145 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:56.146 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:56.146 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:56.146 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:21:56.146 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:21:56.147 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:21:56.147 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:21:56.147 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:21:56.147 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:21:56.147 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:21:56.148 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:21:56.148 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:21:56.148 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:21:56.148 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:56.149 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:56.149 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:21:56.149 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:21:56.149 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:21:56.149 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:21:56.150 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:21:56.150 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:21:56.150 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:21:56.151 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:22:01.152 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:01.153 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:01.154 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:22:01.231 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:22:01.231 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:22:01.777 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:22:01.778 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:22:01.778 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:22:01.778 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:22:01.779 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:22:01.779 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:22:01.779 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:01.780 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:01.780 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:22:01.780 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:01.780 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:22:01.780 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:01.781 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:01.781 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:22:01.781 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:01.781 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:22:01.782 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:01.782 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:22:01.782 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:01.783 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:01.783 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:01.783 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:01.783 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:22:01.784 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:01.784 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:01.784 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:22:01.785 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:01.785 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:01.786 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:01.786 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:22:01.786 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:01.787 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:01.787 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:01.787 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:01.787 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:22:01.787 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:01.788 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:01.788 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:22:01.788 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:01.788 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:01.789 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:01.789 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:22:01.789 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:01.789 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:01.790 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:01.790 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:01.790 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:22:01.790 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:01.790 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:01.791 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:22:01.791 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:01.791 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:01.791 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:01.792 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:22:01.792 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:01.792 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:01.792 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:01.793 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:22:01.793 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:01.793 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:22:01.793 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:22:01.793 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:22:01.794 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:01.794 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:22:01.795 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:22:06.797 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:06.798 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:06.799 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:22:06.915 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:22:06.915 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:22:07.286 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:22:07.287 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:22:07.287 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:22:07.287 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:22:07.287 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:22:07.288 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:22:07.288 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:07.288 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:07.288 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:22:07.288 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:07.289 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:22:07.289 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:07.289 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:07.289 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:22:07.290 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:07.290 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:22:07.290 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:07.290 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:22:07.291 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:07.291 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:07.291 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:07.291 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:07.292 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:22:07.292 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:07.292 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:07.292 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:22:07.293 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:07.293 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:07.293 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:07.293 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:22:07.293 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:07.294 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:07.294 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:07.294 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:07.294 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:22:07.295 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:07.295 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:07.295 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:22:07.295 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:07.296 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:07.296 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:07.296 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:22:07.296 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:07.297 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:07.297 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:07.297 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:07.297 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:22:07.297 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:07.298 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:07.298 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:22:07.298 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:07.298 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:07.299 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:07.299 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:22:07.299 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:07.299 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:07.300 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:07.300 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:22:07.300 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:07.300 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:22:07.301 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:22:07.301 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:22:07.301 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:07.301 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:22:07.303 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:22:12.304 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:12.305 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:12.306 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:22:12.396 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:22:12.396 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:22:12.684 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:22:12.685 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:22:12.685 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:22:12.685 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:22:12.686 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:22:12.686 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:22:12.686 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:12.686 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:12.687 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:22:12.687 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:12.688 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:22:12.688 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:12.688 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:12.688 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:22:12.688 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:12.689 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:22:12.689 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:12.689 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:22:12.689 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:12.690 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:12.690 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:12.690 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:12.690 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:22:12.691 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:12.691 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:12.691 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:22:12.691 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:12.692 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:12.692 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:12.692 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:22:12.692 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:12.692 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:12.693 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:12.693 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:12.693 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:22:12.693 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:12.693 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:12.694 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:22:12.694 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:12.694 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:12.694 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:12.695 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:22:12.695 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:12.695 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:12.695 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:12.696 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:12.696 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:22:12.696 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:12.696 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:12.696 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:22:12.697 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:12.697 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:12.697 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:12.697 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:22:12.698 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:12.698 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:12.698 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:12.698 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:22:12.699 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:12.699 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:22:12.699 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:22:12.699 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:22:12.700 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:12.700 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:22:12.701 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:22:17.703 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:17.704 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:17.705 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:22:17.803 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:22:17.804 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:22:18.252 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:22:18.252 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:22:18.253 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:22:18.253 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:22:18.253 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:22:18.253 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:22:18.254 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:18.254 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:18.254 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:22:18.254 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:18.255 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:22:18.255 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:18.255 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:18.255 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:22:18.256 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:18.256 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:22:18.256 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:18.256 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:22:18.257 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:18.257 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:18.257 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:18.257 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:18.258 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:22:18.258 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:18.258 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:18.258 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:22:18.259 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:18.259 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:18.259 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:18.259 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:22:18.260 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:18.260 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:18.260 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:18.260 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:18.261 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:22:18.261 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:18.261 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:18.261 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.973566_212.1", 2021-11-11T21:22:18.262 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:18.262 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:18.262 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:18.262 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.973566Z", 2021-11-11T21:22:18.263 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:18.263 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:18.263 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:18.263 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:18.263 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:22:18.264 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:18.264 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:18.264 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:22:18.264 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:18.265 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:18.265 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:18.265 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:22:18.265 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:18.266 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:18.266 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:18.266 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:22:18.267 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:18.267 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:22:18.267 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:22:18.267 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:22:18.268 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:18.268 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:22:18.269 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.973566_212.1 2021-11-11T21:22:23.271 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:23.272 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:23.272 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:22:23.397 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:22:23.398 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:22:23.764 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:22:23.764 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:22:23.765 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:22:23.765 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:22:23.765 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:22:23.765 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:22:23.765 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:23.766 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:23.766 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:22:23.766 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:23.766 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:22:23.767 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:23.767 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:23.767 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:22:23.767 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:23.767 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:22:23.768 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:23.768 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:22:23.768 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:23.768 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:23.768 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:23.769 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:23.769 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:22:23.769 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:23.769 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:23.769 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:22:23.770 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:23.770 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:23.770 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:23.770 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:22:23.770 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:23.771 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:23.771 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:23.771 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:23.771 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:22:23.772 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:23.772 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:23.772 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:22:23.772 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:23.773 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:23.773 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:23.773 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:22:23.773 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:23.773 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:23.774 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:23.774 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:23.774 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:22:23.774 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:23.774 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:23.775 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:22:23.775 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:23.775 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:23.775 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:23.775 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:22:23.776 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:23.776 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:23.776 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:23.776 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:22:23.776 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:23.777 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:22:23.777 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:22:23.777 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:22:23.777 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:23.777 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:22:23.778 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:22:28.779 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:28.780 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:28.781 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:22:28.886 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:22:28.887 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:22:29.229 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:22:29.229 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:22:29.229 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:22:29.230 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:22:29.230 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:22:29.230 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:22:29.230 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:29.231 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:29.231 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:22:29.231 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:29.231 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:22:29.232 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:29.232 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:29.232 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:22:29.232 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:29.233 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:22:29.233 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:29.233 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:22:29.233 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:29.234 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:29.234 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:29.234 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:29.234 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:22:29.235 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:29.235 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:29.236 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:22:29.236 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:29.236 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:29.237 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:29.237 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:22:29.238 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:29.238 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:29.239 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:29.239 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:29.239 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:22:29.240 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:29.240 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:29.241 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:22:29.241 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:29.242 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:29.242 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:29.243 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:22:29.243 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:29.244 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:29.244 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:29.245 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:29.245 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:22:29.246 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:29.247 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:29.247 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:22:29.248 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:29.248 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:29.249 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:29.249 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:22:29.250 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:29.250 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:29.251 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:29.251 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:22:29.251 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:29.252 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:22:29.252 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:22:29.253 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:22:29.253 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:29.254 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:22:29.254 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:22:34.257 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:34.257 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:34.258 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:22:34.352 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:22:34.352 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:22:34.820 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:22:34.820 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:22:34.821 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:22:34.821 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:22:34.821 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:22:34.821 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:22:34.822 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:34.822 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:34.822 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:22:34.822 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:34.823 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:22:34.823 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:34.823 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:34.823 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:22:34.823 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:34.824 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:22:34.824 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:34.824 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:22:34.824 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:34.825 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:34.825 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:34.825 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:34.825 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:22:34.825 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:34.826 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:34.826 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:22:34.826 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:34.826 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:34.827 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:34.827 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:22:34.827 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:34.827 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:34.827 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:34.828 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:34.828 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:22:34.828 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:34.828 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:34.828 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:22:34.829 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:34.829 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:34.829 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:34.829 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:22:34.829 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:34.830 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:34.830 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:34.830 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:34.830 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:22:34.830 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:34.831 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:34.831 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:22:34.831 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:34.831 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:34.832 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:34.832 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:22:34.832 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:34.832 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:34.833 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:34.833 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:22:34.833 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:34.833 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:22:34.833 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:22:34.834 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:22:34.834 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:34.834 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:22:34.835 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.831934_205.1 2021-11-11T21:22:39.836 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:39.837 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:39.838 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:22:39.916 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:22:39.916 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:22:40.228 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:22:40.228 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:22:40.229 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:22:40.229 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:22:40.229 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:22:40.229 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:22:40.230 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:40.230 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:40.230 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:22:40.230 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:40.231 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:22:40.231 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:40.231 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:40.231 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:22:40.231 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:40.232 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:22:40.232 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:40.232 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:22:40.232 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:40.233 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:40.233 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:40.233 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:40.233 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:22:40.234 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:40.234 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:40.234 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:22:40.234 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:40.235 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:40.235 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:40.235 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:22:40.235 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:40.235 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:40.236 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:40.236 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:40.236 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:22:40.237 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:40.237 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:40.237 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:22:40.237 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:40.237 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:40.238 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:40.238 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:22:40.238 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:40.238 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:40.239 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:40.239 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:40.239 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:22:40.239 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:40.239 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:40.240 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:22:40.240 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:40.240 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:40.240 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:40.241 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:22:40.241 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:40.241 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:40.242 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:40.242 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:22:40.242 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:40.243 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:22:40.243 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:22:40.244 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:22:40.244 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:40.244 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:22:40.245 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:22:45.247 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:45.247 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:45.248 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:22:45.346 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:22:45.347 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:22:45.850 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:22:45.851 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:22:45.851 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:22:45.851 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:22:45.851 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:22:45.852 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:22:45.852 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:45.852 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:45.852 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:22:45.852 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:45.853 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:22:45.853 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:45.853 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:45.853 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:22:45.854 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:45.854 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:22:45.854 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:45.854 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:22:45.855 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:45.855 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:45.855 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:45.855 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:45.856 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:22:45.856 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:45.856 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:45.856 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:22:45.856 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:45.857 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:45.857 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:45.857 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:22:45.857 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:45.858 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:45.858 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:45.858 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:45.858 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:22:45.859 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:45.859 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:45.859 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:22:45.860 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:45.860 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:45.860 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:45.860 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:22:45.861 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:45.861 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:45.861 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:45.861 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:45.862 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:22:45.862 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:45.862 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:45.862 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:22:45.862 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:45.863 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:45.863 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:45.863 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:22:45.863 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:45.864 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:45.864 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:45.864 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:22:45.864 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:45.865 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:22:45.865 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:22:45.865 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:22:45.865 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:45.865 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:22:45.867 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.830110_204.1 2021-11-11T21:22:50.868 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:50.869 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:50.869 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:22:50.945 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:22:50.945 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:22:51.430 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:22:51.431 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:22:51.431 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:22:51.431 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:22:51.432 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:22:51.432 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:22:51.432 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:51.432 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:51.432 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:22:51.433 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:51.433 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:22:51.433 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:51.433 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:51.433 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:22:51.434 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:51.434 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:22:51.434 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:51.434 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:22:51.435 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:51.435 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:51.435 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:51.435 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:51.435 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:22:51.436 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:51.436 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:51.436 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:22:51.436 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:51.436 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:51.437 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:51.437 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:22:51.437 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:51.437 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:51.438 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:51.438 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:51.438 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:22:51.438 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:51.439 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:51.439 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:22:51.439 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:51.439 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:51.439 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:51.440 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:22:51.440 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:51.440 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:51.440 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:51.441 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:51.441 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:22:51.441 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:51.441 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:51.441 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:22:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:22:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:22:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:22:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:22:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:22:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:51.445 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:22:51.446 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.831934_205.1 2021-11-11T21:22:56.447 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:56.448 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:22:56.449 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:22:56.556 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:22:56.556 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:22:57.122 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:22:57.123 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:22:57.123 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:22:57.123 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:22:57.123 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:22:57.124 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:22:57.124 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:57.124 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:57.124 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:22:57.125 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:57.125 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:22:57.125 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:57.125 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:57.126 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:22:57.126 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:57.126 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:22:57.126 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:57.126 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:22:57.127 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:57.127 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:57.127 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:57.127 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:57.128 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:22:57.128 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:57.128 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:57.128 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:22:57.128 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:57.129 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:57.129 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:57.129 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:22:57.129 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:57.130 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:57.130 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:57.130 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:57.130 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:22:57.131 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:57.131 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:57.131 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:22:57.131 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:57.131 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:57.132 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:57.132 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:22:57.132 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:57.132 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:57.133 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:57.133 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:22:57.133 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:22:57.133 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:22:57.133 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:22:57.134 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:22:57.134 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:22:57.134 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:22:57.134 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:22:57.135 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:22:57.135 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:22:57.135 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:57.135 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:57.135 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:22:57.136 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:22:57.136 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:22:57.136 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:22:57.136 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:22:57.137 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:22:57.137 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:22:57.138 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.761008_196.1 2021-11-11T21:23:02.140 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:02.141 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:02.142 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:23:02.230 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:23:02.231 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:23:02.698 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:23:02.698 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:23:02.698 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:23:02.699 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:23:02.699 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:23:02.699 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:23:02.699 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:02.700 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:02.700 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:23:02.700 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:02.700 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:23:02.701 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:02.701 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:02.701 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:23:02.701 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:02.702 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:23:02.702 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:02.702 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:23:02.702 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:02.702 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:02.703 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:02.703 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:02.703 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:23:02.703 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:02.704 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:02.704 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:23:02.704 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:02.704 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:02.704 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:02.705 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:23:02.705 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:02.705 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:02.705 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:02.706 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:02.706 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:23:02.706 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:02.706 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:02.706 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:23:02.707 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:02.707 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:02.707 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:02.707 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:23:02.708 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:02.708 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:02.708 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:02.708 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:02.708 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:23:02.709 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:02.709 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:02.709 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:23:02.709 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:02.710 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:02.710 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:02.710 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:23:02.710 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:02.711 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:02.711 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:02.711 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:23:02.711 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:02.711 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:23:02.712 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:23:02.712 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:23:02.712 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:02.712 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:23:02.714 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:23:07.715 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:07.716 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:07.717 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:23:07.807 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:23:07.807 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:23:08.147 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:23:08.148 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:23:08.148 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:23:08.148 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:23:08.148 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:23:08.149 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:23:08.149 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:08.149 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:08.149 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:23:08.150 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:08.150 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:23:08.150 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:08.150 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:08.151 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:23:08.151 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:08.151 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:23:08.151 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:08.152 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:23:08.152 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:08.152 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:08.152 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:08.152 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:08.153 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:23:08.153 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:08.153 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:08.153 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:23:08.154 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:08.154 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:08.154 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:08.154 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:23:08.154 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:08.155 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:08.155 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:08.155 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:08.156 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:23:08.156 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:08.156 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:08.156 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:23:08.157 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:08.157 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:08.157 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:08.157 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:23:08.158 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:08.158 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:08.158 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:08.158 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:08.159 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:23:08.159 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:08.159 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:08.160 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:23:08.160 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:08.160 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:08.161 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:08.161 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:23:08.161 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:08.161 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:08.162 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:08.162 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:23:08.162 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:08.162 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:23:08.162 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:23:08.163 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:23:08.163 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:08.163 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:23:08.164 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:23:13.166 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:13.167 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:13.167 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:23:13.257 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:23:13.257 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:23:13.746 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:23:13.747 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:23:13.747 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:23:13.747 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:23:13.747 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:23:13.748 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:23:13.748 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:13.748 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:13.748 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:23:13.749 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:13.749 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:23:13.749 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:13.749 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:13.750 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:23:13.750 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:13.750 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:23:13.750 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:13.751 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:23:13.751 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:13.751 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:13.751 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:13.752 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:13.752 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:23:13.752 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:13.752 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:13.753 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:23:13.753 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:13.753 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:13.753 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:13.753 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:23:13.754 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:13.754 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:13.754 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:13.754 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:13.755 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:23:13.755 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:13.755 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:13.755 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:23:13.756 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:13.756 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:13.756 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:13.757 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:23:13.757 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:13.757 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:13.757 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:13.758 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:13.758 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:23:13.758 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:13.758 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:13.759 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:23:13.759 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:13.759 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:13.759 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:13.760 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:23:13.760 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:13.760 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:13.761 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:13.761 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:23:13.761 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:13.761 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:23:13.761 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:23:13.762 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:23:13.762 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:13.762 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:23:13.763 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:23:18.764 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:18.765 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:18.765 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:23:18.865 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:23:18.865 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:23:19.325 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:23:19.325 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:23:19.325 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:23:19.326 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:23:19.326 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:23:19.326 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:23:19.326 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:19.327 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:19.327 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:23:19.327 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:19.327 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:23:19.328 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:19.328 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:19.328 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:23:19.328 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:19.329 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:23:19.329 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:19.330 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:23:19.330 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:19.330 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:19.330 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:19.331 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:19.331 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:23:19.331 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:19.331 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:19.331 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:23:19.332 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:19.332 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:19.332 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:19.332 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:23:19.333 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:19.333 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:19.333 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:19.333 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:19.333 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:23:19.334 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:19.334 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:19.334 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:23:19.334 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:19.335 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:19.335 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:19.335 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:23:19.336 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:19.336 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:19.336 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:19.336 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:19.337 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:23:19.337 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:19.337 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:19.337 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:23:19.337 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:19.338 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:19.338 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:19.338 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:23:19.339 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:19.339 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:19.339 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:19.339 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:23:19.339 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:19.340 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:23:19.340 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:23:19.340 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:23:19.341 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:19.341 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:23:19.343 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.831934_205.1 2021-11-11T21:23:22.316 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:23:22.316+0000 7fb6b95f5700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-5) 2021-11-11T21:23:24.344 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:24.345 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:24.346 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:23:24.445 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:23:24.445 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:23:24.792 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:23:24.793 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:23:24.793 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:23:24.793 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:23:24.793 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:23:24.794 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:23:24.794 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:24.794 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:24.794 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:23:24.795 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:24.795 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:23:24.795 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:24.795 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:24.796 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:23:24.796 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:24.796 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:23:24.796 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:24.797 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:23:24.797 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:24.797 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:24.797 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:24.798 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:24.798 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:23:24.798 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:24.798 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:24.799 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:23:24.799 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:24.799 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:24.799 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:24.799 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:23:24.800 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:24.800 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:24.801 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:24.801 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:24.801 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:23:24.801 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:24.802 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:24.802 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:23:24.802 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:24.802 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:24.803 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:24.803 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:23:24.803 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:24.803 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:24.804 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:24.804 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:24.804 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:23:24.804 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:24.805 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:24.805 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:23:24.805 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:24.805 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:24.806 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:24.806 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:23:24.806 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:24.807 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:24.807 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:24.807 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:23:24.807 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:24.807 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:23:24.808 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:23:24.808 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:23:24.808 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:24.808 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:23:24.810 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:23:29.811 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:29.812 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:29.812 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:23:29.913 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:23:29.914 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:23:30.311 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:23:30.311 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:23:30.311 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:23:30.312 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:23:30.312 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:23:30.312 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:23:30.312 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:30.312 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:30.313 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:23:30.313 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:30.313 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:23:30.313 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:30.314 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:30.314 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:23:30.314 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:30.314 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:23:30.314 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:30.315 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:23:30.315 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:30.315 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:30.315 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:30.315 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:30.316 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:23:30.316 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:30.316 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:30.316 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:23:30.317 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:30.317 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:30.317 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:30.317 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:23:30.317 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:30.318 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:30.322 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:30.322 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:30.323 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:23:30.323 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:30.323 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:30.323 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:23:30.323 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:30.324 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:30.324 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:30.324 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:23:30.324 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:30.325 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:30.325 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:30.325 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:30.325 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:23:30.325 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:30.326 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:30.326 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:23:30.326 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:30.326 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:30.327 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:30.327 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:23:30.327 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:30.327 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:30.327 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:30.328 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:23:30.328 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:30.328 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:23:30.328 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:23:30.329 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:23:30.329 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:30.329 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:23:30.330 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.768842_198.1 2021-11-11T21:23:35.331 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:35.332 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:35.332 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:23:35.442 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:23:35.442 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:23:35.990 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:23:35.990 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:23:35.991 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:23:35.991 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:23:35.992 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:23:35.992 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:23:35.993 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:35.993 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:35.993 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:23:35.994 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:35.994 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:23:35.995 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:35.995 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:35.996 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:23:35.996 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:35.996 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:23:35.997 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:35.997 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:23:35.998 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:35.998 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:35.999 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:35.999 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:35.999 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:23:36.000 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:36.000 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:36.001 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:23:36.001 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:36.002 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:36.002 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:36.002 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:23:36.003 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:36.003 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:36.004 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:36.004 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:36.004 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:23:36.005 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:36.005 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:36.006 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:23:36.006 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:36.006 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:36.007 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:36.007 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:23:36.008 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:36.008 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:36.009 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:36.009 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:36.009 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:23:36.010 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:36.010 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:36.011 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:23:36.011 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:36.011 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:36.012 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:36.012 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:23:36.013 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:36.013 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:36.013 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:36.014 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:23:36.014 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:36.015 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:23:36.015 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:23:36.016 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:23:36.016 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:36.016 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:23:36.018 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:23:41.020 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:41.021 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:41.022 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:23:41.100 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:23:41.101 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:23:41.603 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:23:41.603 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:23:41.603 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:23:41.604 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:23:41.604 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:23:41.604 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:23:41.604 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:41.605 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:41.605 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:23:41.605 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:41.605 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:23:41.605 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:41.606 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:41.606 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:23:41.606 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:41.606 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:23:41.607 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:41.607 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:23:41.607 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:41.607 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:41.608 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:41.608 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:41.608 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:23:41.608 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:41.609 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:41.609 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:23:41.609 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:41.609 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:41.609 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:41.610 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:23:41.610 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:41.610 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:41.610 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:41.611 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:41.611 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:23:41.611 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:41.611 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:41.611 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:23:41.612 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:41.612 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:41.612 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:41.612 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:23:41.612 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:41.613 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:41.613 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:41.613 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:41.613 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:23:41.614 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:41.614 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:41.614 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:23:41.614 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:41.615 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:41.615 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:41.615 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:23:41.615 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:41.615 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:41.616 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:41.616 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:23:41.616 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:41.616 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:23:41.616 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:23:41.617 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:23:41.617 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:41.617 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:23:41.618 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.831934_205.1 2021-11-11T21:23:42.427 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:23:42.426+0000 7fcab3609700 -1 ERROR: failed operation: ERROR: collect() returned error (ret=-5) 2021-11-11T21:23:46.620 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:46.621 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:46.621 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:23:46.696 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:23:46.697 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:23:47.093 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:23:47.093 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:23:47.094 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:23:47.094 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:23:47.094 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:23:47.094 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:23:47.095 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:47.095 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:47.095 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:23:47.095 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:47.095 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:23:47.096 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:47.096 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:47.096 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:23:47.096 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:47.097 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:23:47.097 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:47.097 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:23:47.097 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:47.098 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:47.098 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:47.098 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:47.098 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:23:47.098 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:47.099 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:47.099 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:23:47.099 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:47.099 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:47.100 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:47.100 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:23:47.100 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:47.100 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:47.101 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:47.101 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:47.101 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:23:47.101 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:47.101 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:47.102 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:23:47.102 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:47.102 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:47.102 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:47.103 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:23:47.103 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:47.103 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:47.103 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:47.104 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:47.104 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:23:47.104 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:47.104 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:47.104 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:23:47.105 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:47.105 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:47.105 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:47.106 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:23:47.106 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:47.106 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:47.107 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:47.107 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:23:47.107 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:47.107 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:23:47.107 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:23:47.108 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:23:47.108 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:47.108 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:23:47.110 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:23:52.111 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:52.112 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:52.112 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:23:52.203 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:23:52.203 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:23:52.614 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:23:52.615 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:23:52.615 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:23:52.615 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:23:52.615 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:23:52.615 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:23:52.616 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:52.616 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:52.616 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:23:52.616 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:52.616 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:23:52.617 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:52.617 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:52.617 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:23:52.617 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:52.617 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:23:52.618 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:52.618 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:23:52.618 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:52.618 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:52.619 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:52.619 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:52.619 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:23:52.619 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:52.619 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:52.620 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:23:52.620 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:52.620 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:52.620 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:52.621 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:23:52.621 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:52.621 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:52.621 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:52.621 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:52.622 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:23:52.622 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:52.622 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:52.622 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:23:52.623 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:52.623 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:52.623 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:52.623 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:23:52.623 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:52.624 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:52.624 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:52.624 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:52.624 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:23:52.625 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:52.625 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:52.625 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:23:52.625 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:52.625 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:52.626 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:52.626 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:23:52.626 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:52.626 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:52.627 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:52.627 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:23:52.627 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:52.627 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:23:52.628 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:23:52.628 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:23:52.628 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:52.628 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:23:52.629 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.783136_199.1 2021-11-11T21:23:57.630 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:57.631 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:23:57.632 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:23:57.729 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:23:57.730 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:23:58.016 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:23:58.016 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:23:58.016 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:23:58.017 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:23:58.017 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:23:58.017 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:23:58.017 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:58.018 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:58.018 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:23:58.018 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:58.019 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:23:58.019 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:58.019 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:58.019 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:23:58.020 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:58.020 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:23:58.020 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:58.020 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:23:58.020 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:58.021 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:58.021 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:58.021 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:58.021 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:23:58.022 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:58.022 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:58.022 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:23:58.022 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:58.023 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:58.023 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:58.023 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:23:58.023 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:58.023 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:58.024 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:58.024 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:58.024 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:23:58.024 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:58.025 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:58.025 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:23:58.025 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:58.025 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:58.025 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:58.026 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:23:58.026 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:58.026 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:58.026 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:58.027 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:23:58.027 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:23:58.027 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:23:58.027 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:23:58.028 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:23:58.028 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:23:58.028 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:23:58.028 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:23:58.028 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:23:58.029 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:23:58.029 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:58.029 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:58.029 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:23:58.030 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:23:58.030 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:23:58.030 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:23:58.030 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:23:58.031 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:23:58.031 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:23:58.031 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.789745_201.1 2021-11-11T21:24:03.033 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:03.034 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:03.034 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:03.124 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:03.124 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:03.358 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:24:03.359 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:24:03.359 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:24:03.359 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:24:03.359 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:24:03.359 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:24:03.360 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:03.360 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:03.360 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:24:03.360 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:03.361 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:24:03.361 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:03.361 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:03.361 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:03.362 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:03.362 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:24:03.362 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:03.362 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:24:03.363 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:03.363 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:03.363 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:03.363 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:03.363 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:24:03.364 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:03.364 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:03.364 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:24:03.364 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:03.365 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:03.365 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:03.365 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:24:03.365 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:03.365 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:03.366 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:03.366 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:03.366 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:24:03.366 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:03.367 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:03.367 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:24:03.367 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:03.367 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:03.368 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:03.368 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:24:03.368 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:03.368 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:03.368 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:03.369 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:03.369 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:24:03.369 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:03.369 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:03.369 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:03.370 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:03.370 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:03.370 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:03.370 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:24:03.371 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:03.371 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:03.371 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:03.371 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:24:03.372 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:03.372 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:24:03.372 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:24:03.372 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:24:03.372 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:03.373 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:24:03.374 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.830110_204.1 2021-11-11T21:24:08.375 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:08.376 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:08.376 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:08.469 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:08.469 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:08.812 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:24:08.813 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:24:08.813 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:24:08.813 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:24:08.814 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:24:08.814 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:24:08.814 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:08.814 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:08.815 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:24:08.815 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:08.815 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:24:08.815 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:08.816 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:08.816 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:08.816 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:08.816 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:24:08.817 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:08.817 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:24:08.817 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:08.817 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:08.818 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:08.818 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:08.818 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:24:08.818 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:08.819 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:08.819 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:24:08.819 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:08.819 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:08.819 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:08.820 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:24:08.820 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:08.820 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:08.820 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:08.821 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:08.821 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:24:08.821 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:08.821 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:08.822 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:24:08.822 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:08.822 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:08.822 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:08.822 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:24:08.823 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:08.823 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:08.823 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:08.823 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:08.824 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:24:08.824 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:08.824 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:08.824 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:08.824 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:08.825 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:08.825 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:08.825 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:24:08.825 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:08.826 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:08.826 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:08.826 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:24:08.826 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:08.826 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:24:08.827 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:24:08.827 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:24:08.827 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:08.827 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:24:08.828 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.830110_204.1 2021-11-11T21:24:13.829 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:13.830 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:13.831 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:13.912 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:13.912 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:14.237 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:24:14.238 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:24:14.238 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:24:14.238 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:24:14.239 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:24:14.239 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:24:14.239 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:14.239 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:14.240 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:24:14.240 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:14.240 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:24:14.240 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:14.241 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:14.241 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:14.241 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:14.241 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:24:14.241 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:14.242 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:24:14.242 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:14.242 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:14.242 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:14.243 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:14.243 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:24:14.243 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:14.243 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:14.243 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:24:14.244 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:14.244 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:14.244 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:14.244 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:24:14.245 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:14.245 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:14.245 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:14.245 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:14.246 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:24:14.246 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:14.246 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:14.246 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:24:14.246 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:14.247 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:14.247 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:14.247 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:24:14.247 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:14.248 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:14.248 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:14.248 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:14.248 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:24:14.248 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:14.249 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:14.249 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:14.249 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:14.249 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:14.250 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:14.250 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:24:14.250 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:14.250 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:14.250 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:14.251 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:24:14.251 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:14.251 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:24:14.251 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:24:14.252 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:24:14.252 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:14.252 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:24:14.253 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:24:19.254 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:19.255 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:19.256 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:19.337 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:19.338 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:19.809 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:24:19.809 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:24:19.810 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:24:19.810 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:24:19.810 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:24:19.810 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:24:19.810 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:19.811 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:19.811 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:24:19.811 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:19.811 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:24:19.811 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:19.812 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:19.812 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:19.812 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:19.812 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:24:19.812 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:19.813 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:24:19.813 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:19.813 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:19.813 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:19.813 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:19.814 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:24:19.814 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:19.814 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:19.814 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:24:19.815 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:19.815 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:19.815 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:19.815 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:24:19.815 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:19.816 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:19.816 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:19.816 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:19.816 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:24:19.816 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:19.817 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:19.817 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:24:19.817 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:19.817 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:19.818 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:19.818 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:24:19.818 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:19.818 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:19.818 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:19.819 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:19.819 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:24:19.819 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:19.819 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:19.819 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:19.820 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:19.820 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:19.820 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:19.820 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:24:19.821 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:19.821 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:19.821 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:19.821 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:24:19.822 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:19.822 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:24:19.822 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:24:19.822 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:24:19.822 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:19.823 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:24:19.823 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.762816_197.1 2021-11-11T21:24:24.825 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:24.826 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:24.826 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:24.920 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:24.921 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:25.211 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:24:25.211 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:24:25.211 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:24:25.211 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:24:25.212 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:24:25.212 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:24:25.212 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:25.212 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:25.213 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:24:25.213 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:25.213 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:24:25.213 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:25.213 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:25.214 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:25.214 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:25.214 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:24:25.214 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:25.215 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:24:25.215 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:25.215 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:25.215 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:25.215 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:25.216 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:24:25.216 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:25.216 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:25.216 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:24:25.217 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:25.217 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:25.217 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:25.217 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:24:25.217 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:25.218 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:25.218 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:25.218 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:25.218 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:24:25.219 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:25.219 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:25.219 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:24:25.219 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:25.219 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:25.220 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:25.220 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:24:25.220 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:25.220 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:25.220 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:25.221 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:25.221 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:24:25.221 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:25.221 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:25.222 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:25.222 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:25.222 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:25.222 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:25.222 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:24:25.223 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:25.223 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:25.223 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:25.223 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:24:25.224 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:25.224 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:24:25.224 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:24:25.224 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:24:25.224 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:25.225 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:24:25.226 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665539.417369_286.1 target=1_1636665224.783136_199.1 2021-11-11T21:24:30.228 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_multi_period_incremental_sync ... FAIL 2021-11-11T21:24:30.229 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-42 2021-11-11T21:24:30.267 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-43 2021-11-11T21:24:31.563 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-44 2021-11-11T21:24:31.706 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:24:31.706+0000 7fb6d4244700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:24:31.707 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:24:31.706+0000 7fcac6a49700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:24:31.707 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:24:31.707+0000 7fb6d123e700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:24:31.819 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:31.820 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:31.820 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:31.949 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:31.949 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:32.001 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:24:32.035 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:24:32.035 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:24:32.035 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:32.036 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:24:32.036 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:24:32.036 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:24:32.036 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:24:32.037 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:24:32.037 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:24:32.037 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:24:32.037 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665871.647321_298.1", 2021-11-11T21:24:32.038 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:24:31.647321Z" 2021-11-11T21:24:32.038 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:24:32.038 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:24:32.038 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:32.039 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:24:32.039 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:24:32.039 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:24:32.040 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:24:32.041 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:32.041 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:32.042 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:32.123 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:32.123 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:32.771 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:24:32.771+0000 7fb6c722a700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:24:32.800 INFO:tasks.rgw.c2.client.0.smithi198.stdout:2021-11-11T21:24:32.799+0000 7fcacba53700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:24:32.812 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:24:32.812 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:24:32.813 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:24:32.813 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:24:32.813 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:24:32.813 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:24:32.814 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:32.814 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:32.814 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:24:32.814 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:32.815 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:24:32.815 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:32.815 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:32.815 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:32.815 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:32.816 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:24:32.816 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:32.816 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:24:32.816 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:32.817 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:32.817 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:32.817 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:32.817 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:24:32.817 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:32.818 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:32.818 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:24:32.818 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:32.818 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:32.819 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:32.819 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:24:32.819 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:32.819 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:32.820 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:32.820 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:32.820 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:24:32.820 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:32.820 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:32.821 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:24:32.821 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:32.821 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:32.821 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:32.822 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:24:32.822 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:32.822 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:32.822 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:32.822 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:32.823 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:24:32.823 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:32.823 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:32.823 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:32.824 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:32.824 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:32.824 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:32.824 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:24:32.824 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:32.825 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:32.825 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:32.825 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:24:32.825 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:32.825 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:24:32.826 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:24:32.826 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:24:32.826 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:32.826 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:24:32.828 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.831934_205.1 2021-11-11T21:24:32.907 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:24:32.907+0000 7f04fac32700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:24:32.908 INFO:tasks.rgw.c2.client.1.smithi198.stdout:2021-11-11T21:24:32.908+0000 7f04fe439700 -1 rgw async rados processor: ERROR: _send_request(): get_sync_policy_handler() returned -2 2021-11-11T21:24:37.829 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:37.830 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:37.831 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:37.873 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:37.873 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:37.939 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:24:37.939 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:24:37.939 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:24:37.940 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:24:37.940 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:24:37.940 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:24:37.940 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:37.941 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:37.941 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:24:37.941 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:37.941 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:24:37.941 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:37.942 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:37.942 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:37.942 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:37.943 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:24:37.943 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:37.943 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:24:37.944 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:37.944 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:37.944 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:37.944 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:37.944 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:24:37.945 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:37.945 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:37.945 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:24:37.945 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:37.946 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:37.946 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:37.946 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:24:37.946 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:37.947 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:37.947 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:37.947 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:37.947 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:24:37.947 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:37.948 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:37.948 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:24:37.948 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:37.948 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:37.949 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:37.949 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:24:37.949 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:37.949 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:37.950 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:37.950 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:37.950 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:24:37.950 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:37.950 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:37.952 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:37.952 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:37.952 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:37.952 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:37.953 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:24:37.953 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:37.953 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:37.953 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:37.953 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:24:37.954 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:37.954 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:24:37.954 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:24:37.954 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:24:37.955 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:37.955 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:24:37.956 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.768842_198.1 2021-11-11T21:24:42.957 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:42.958 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:42.959 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:43.010 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:43.010 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:43.082 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:24:43.083 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:24:43.083 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:24:43.083 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:24:43.083 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:24:43.084 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:24:43.084 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:43.084 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:43.084 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:24:43.085 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:43.085 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:24:43.085 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:43.085 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:43.086 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:43.086 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:43.086 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:24:43.086 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:43.087 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:24:43.087 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:43.087 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:43.087 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:43.088 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:43.088 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:24:43.088 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:43.088 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:43.088 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:24:43.089 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:43.089 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:43.089 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:43.089 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:24:43.090 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:43.090 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:43.090 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:43.090 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:43.091 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:24:43.091 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:43.091 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:43.091 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.914354_210.1", 2021-11-11T21:24:43.091 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:43.092 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:43.092 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:43.092 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.914354Z", 2021-11-11T21:24:43.092 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:43.093 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:43.093 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:43.093 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:43.093 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:24:43.093 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:43.094 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:43.094 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:43.094 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:43.094 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:43.095 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:43.095 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:24:43.095 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:43.095 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:43.096 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:43.096 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:24:43.096 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:43.096 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:24:43.096 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:24:43.097 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:24:43.097 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:43.097 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:24:43.098 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.914354_210.1 2021-11-11T21:24:48.100 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:48.100 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:48.101 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:48.180 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:48.181 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:48.322 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:24:48.323 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:24:48.323 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:24:48.323 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:24:48.324 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:24:48.324 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:24:48.324 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:48.324 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:48.324 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:24:48.325 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:48.325 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:24:48.325 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:48.325 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:48.326 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:48.326 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:48.326 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:24:48.326 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:48.326 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:24:48.327 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:48.327 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:48.327 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:48.327 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:48.328 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:24:48.328 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:48.328 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:48.328 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:24:48.329 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:48.329 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:48.329 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:48.329 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:24:48.329 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:48.330 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:48.330 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:48.330 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:48.330 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:24:48.331 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:48.331 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:48.331 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:24:48.331 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:48.331 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:48.332 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:48.332 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:24:48.332 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:48.332 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:48.332 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:48.333 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:48.333 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:24:48.333 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:48.333 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:48.334 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:48.334 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:48.334 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:48.334 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:48.334 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:24:48.335 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:48.335 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:48.335 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:48.335 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:24:48.336 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:48.336 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:24:48.336 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:24:48.336 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:24:48.336 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:48.337 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:24:48.338 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.789745_201.1 2021-11-11T21:24:53.339 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:53.340 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:53.340 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:53.388 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:53.388 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:53.836 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:24:53.837 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:24:53.837 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:24:53.838 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:24:53.838 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:24:53.838 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:24:53.839 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:53.839 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:53.840 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:24:53.840 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:53.840 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:24:53.841 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:53.841 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:53.842 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:53.842 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:53.842 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:24:53.843 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:53.843 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:24:53.844 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:53.844 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:53.844 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:53.845 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:53.845 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:24:53.845 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:53.845 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:53.845 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:24:53.846 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:53.846 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:53.846 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:53.846 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:24:53.847 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:53.847 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:53.847 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:53.847 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:53.847 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:24:53.848 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:53.848 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:53.848 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:24:53.848 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:53.849 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:53.849 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:53.849 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:24:53.850 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:53.850 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:53.850 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:53.851 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:53.851 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:24:53.851 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:53.851 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:53.852 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:53.852 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:53.852 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:53.852 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:53.852 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:24:53.853 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:53.853 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:53.853 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:53.853 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:24:53.854 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:53.854 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:24:53.854 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:24:53.854 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:24:53.854 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:53.855 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:24:53.857 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.825641_203.1 2021-11-11T21:24:58.858 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:58.859 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:24:58.860 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:24:58.949 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:24:58.949 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:24:59.422 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:24:59.423 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:24:59.423 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:24:59.423 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:24:59.423 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:24:59.424 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:24:59.424 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:59.424 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:59.424 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:24:59.424 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:59.425 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:24:59.425 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:59.425 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:59.425 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:24:59.426 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:59.426 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:24:59.426 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:59.426 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:24:59.427 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:59.427 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:59.427 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:59.427 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:59.427 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:24:59.428 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:59.428 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:59.428 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:24:59.428 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:59.429 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:59.429 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:59.429 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:24:59.429 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:59.430 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:59.430 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:59.430 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:59.430 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:24:59.431 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:59.431 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:59.431 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:24:59.431 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:59.432 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:59.432 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:59.432 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:24:59.432 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:59.433 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:59.433 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:59.433 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:24:59.433 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:24:59.433 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:24:59.434 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:24:59.434 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:24:59.434 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:24:59.434 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:24:59.435 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:24:59.435 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:24:59.435 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:24:59.435 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:59.435 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:59.436 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:24:59.436 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:24:59.436 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:24:59.436 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:24:59.437 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:24:59.437 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:24:59.438 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:24:59.439 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.787834_200.1 2021-11-11T21:25:04.440 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:04.441 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:04.442 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:25:04.491 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:25:04.491 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:25:04.610 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:25:04.610 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:25:04.611 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:25:04.611 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:25:04.611 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:25:04.611 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:25:04.612 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:04.612 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:04.612 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:25:04.612 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:04.613 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:25:04.613 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:04.613 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:04.613 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:25:04.614 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:04.614 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:25:04.614 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:04.614 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:25:04.614 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:04.615 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:04.615 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:04.615 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:04.615 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:25:04.616 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:04.616 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:04.616 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:25:04.616 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:04.617 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:04.617 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:04.617 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:25:04.617 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:04.617 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:04.618 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:04.618 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:04.618 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:25:04.618 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:04.619 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:04.619 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:25:04.619 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:04.619 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:04.619 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:04.620 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:25:04.620 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:04.620 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:04.620 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:04.621 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:04.621 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:25:04.621 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:04.621 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:04.621 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:25:04.622 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:04.622 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:04.622 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:04.622 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:25:04.623 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:04.623 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:04.623 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:04.623 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:25:04.624 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:04.624 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:25:04.624 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:25:04.624 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:25:04.625 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:04.625 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:25:04.626 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.789745_201.1 2021-11-11T21:25:09.627 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:09.628 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:09.629 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:25:09.698 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:25:09.699 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:25:09.766 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:25:09.767 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:25:09.767 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:25:09.767 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:25:09.768 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:25:09.768 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:25:09.768 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:09.768 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:09.769 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:25:09.769 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:09.769 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:25:09.769 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:09.769 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:09.770 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:25:09.770 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:09.770 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:25:09.770 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:09.771 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:25:09.771 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:09.771 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:09.771 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:09.771 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:09.772 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:25:09.772 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:09.772 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:09.772 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:25:09.773 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:09.773 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:09.773 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:09.773 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:25:09.773 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:09.774 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:09.774 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:09.774 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:09.774 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:25:09.775 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:09.775 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:09.775 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:25:09.775 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:09.775 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:09.776 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:09.776 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:25:09.776 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:09.776 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:09.776 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:09.777 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:09.777 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:25:09.777 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:09.777 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:09.778 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:25:09.778 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:09.778 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:09.778 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:09.778 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:25:09.779 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:09.779 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:09.779 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:09.779 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:25:09.780 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:09.780 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:25:09.780 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:25:09.780 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:25:09.780 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:09.781 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:25:09.782 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.783136_199.1 2021-11-11T21:25:14.783 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:14.784 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:14.785 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:25:14.827 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:25:14.828 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:25:14.917 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:25:14.917 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:25:14.917 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:25:14.918 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:25:14.918 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:25:14.918 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:25:14.918 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:14.919 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:14.919 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:25:14.919 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:14.920 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:25:14.920 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:14.920 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:14.921 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:25:14.921 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:14.921 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:25:14.921 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:14.921 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:25:14.922 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:14.922 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:14.922 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:14.922 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:14.923 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:25:14.923 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:14.923 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:14.923 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:25:14.924 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:14.924 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:14.924 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:14.924 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:25:14.924 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:14.925 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:14.925 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:14.925 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:14.925 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:25:14.926 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:14.926 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:14.926 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:25:14.927 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:14.927 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:14.927 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:14.927 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:25:14.928 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:14.928 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:14.928 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:14.928 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:14.929 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:25:14.929 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:14.929 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:14.929 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:25:14.930 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:14.930 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:14.930 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:14.930 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:25:14.930 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:14.931 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:14.931 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:14.931 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:25:14.931 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:14.932 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:25:14.932 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:25:14.932 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:25:14.932 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:14.933 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:25:14.934 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.831934_205.1 2021-11-11T21:25:19.936 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:19.937 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:19.937 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:25:20.013 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:25:20.014 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:25:20.447 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:25:20.448 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:25:20.448 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:25:20.448 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:25:20.449 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:25:20.449 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:25:20.449 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:20.449 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:20.450 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:25:20.450 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:20.450 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:25:20.450 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:20.451 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:20.451 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:25:20.451 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:20.451 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:25:20.452 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:20.452 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:25:20.453 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:20.453 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:20.453 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:20.453 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:20.454 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:25:20.454 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:20.454 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:20.454 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:25:20.456 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:20.456 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:20.456 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:20.457 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:25:20.457 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:20.457 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:20.457 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:20.457 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:20.458 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:25:20.458 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:20.458 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:20.458 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:25:20.459 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:20.459 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:20.459 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:20.459 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:25:20.460 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:20.460 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:20.460 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:20.460 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:20.461 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:25:20.461 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:20.461 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:20.461 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:25:20.462 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:20.462 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:20.462 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:20.463 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:25:20.463 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:20.463 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:20.463 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:20.464 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:25:20.464 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:20.464 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:25:20.464 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:25:20.464 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:25:20.465 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:20.465 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:25:20.465 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.761008_196.1 2021-11-11T21:25:25.467 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:25.468 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:25.469 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:25:25.539 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:25:25.539 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:25:25.620 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:25:25.620 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:25:25.620 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:25:25.621 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:25:25.621 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:25:25.621 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:25:25.621 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:25.622 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:25.622 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:25:25.622 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:25.622 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:25:25.623 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:25.623 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:25.623 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:25:25.623 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:25.623 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:25:25.624 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:25.624 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:25:25.624 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:25.625 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:25.625 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:25.625 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:25.625 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:25:25.625 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:25.626 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:25.626 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:25:25.626 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:25.626 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:25.627 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:25.627 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:25:25.627 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:25.627 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:25.627 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:25.628 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:25.628 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:25:25.628 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:25.628 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:25.628 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:25:25.629 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:25.630 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:25.630 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:25.630 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:25:25.631 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:25.631 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:25.631 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:25.631 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:25.632 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:25:25.632 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:25.632 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:25.632 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:25:25.633 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:25.633 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:25.633 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:25.633 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:25:25.634 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:25.634 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:25.634 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:25.634 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:25:25.635 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:25.635 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:25:25.635 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:25:25.635 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:25:25.635 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:25.636 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:25:25.637 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.831934_205.1 2021-11-11T21:25:30.638 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:30.639 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:30.639 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:25:30.701 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:25:30.701 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:25:31.164 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:25:31.165 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:25:31.165 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:25:31.166 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:25:31.166 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:25:31.167 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:25:31.167 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:31.167 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:31.168 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:25:31.168 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:31.168 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:25:31.169 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:31.169 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:31.169 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:25:31.169 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:31.170 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:25:31.170 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:31.170 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:25:31.170 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:31.171 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:31.171 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:31.171 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:31.171 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:25:31.172 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:31.172 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:31.172 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:25:31.172 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:31.173 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:31.173 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:31.173 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:25:31.173 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:31.174 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:31.174 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:31.174 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:31.174 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:25:31.175 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:31.175 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:31.175 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:25:31.175 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:31.176 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:31.176 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:31.176 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:25:31.176 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:31.176 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:31.177 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:31.177 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:31.177 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:25:31.178 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:31.178 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:31.178 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:25:31.178 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:31.179 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:31.179 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:31.179 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:25:31.180 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:31.180 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:31.180 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:31.181 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:25:31.181 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:31.181 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:25:31.181 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:25:31.182 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:25:31.182 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:31.182 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:25:31.184 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:25:36.186 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:36.187 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:36.187 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:25:36.238 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:25:36.238 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:25:36.491 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:25:36.492 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:25:36.492 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:25:36.492 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:25:36.492 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:25:36.493 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:25:36.493 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:36.493 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:36.493 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:25:36.493 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:36.494 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:25:36.494 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:36.494 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:36.494 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:25:36.495 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:36.495 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:25:36.495 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:36.495 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:25:36.495 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:36.496 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:36.496 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:36.496 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:36.496 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:25:36.497 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:36.497 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:36.497 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:25:36.497 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:36.498 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:36.498 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:36.498 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:25:36.498 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:36.498 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:36.499 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:36.499 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:36.499 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:25:36.499 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:36.499 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:36.500 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:25:36.500 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:36.500 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:36.500 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:36.501 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:25:36.501 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:36.501 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:36.501 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:36.501 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:36.502 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:25:36.502 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:36.502 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:36.502 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:25:36.503 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:36.503 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:36.503 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:36.503 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:25:36.503 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:36.504 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:36.504 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:36.504 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:25:36.504 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:36.505 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:25:36.505 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:25:36.505 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:25:36.505 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:36.505 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:25:36.506 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.761008_196.1 2021-11-11T21:25:41.507 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:41.508 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:41.508 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:25:41.555 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:25:41.556 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:25:41.627 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:25:41.627 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:25:41.628 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:25:41.628 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:25:41.629 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:25:41.629 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:25:41.629 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:41.629 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:41.630 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:25:41.630 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:41.630 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:25:41.631 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:41.631 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:41.631 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:25:41.631 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:41.632 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:25:41.632 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:41.632 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:25:41.633 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:41.633 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:41.633 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:41.633 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:41.633 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:25:41.634 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:41.634 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:41.634 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:25:41.634 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:41.635 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:41.635 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:41.635 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:25:41.635 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:41.636 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:41.636 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:41.636 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:41.636 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:25:41.637 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:41.637 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:41.637 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:25:41.637 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:41.637 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:41.638 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:41.638 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:25:41.638 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:41.638 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:41.639 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:41.639 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:41.639 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:25:41.639 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:41.640 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:41.640 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:25:41.640 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:41.640 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:41.641 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:41.641 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:25:41.642 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:41.642 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:41.642 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:41.642 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:25:41.643 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:41.643 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:25:41.643 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:25:41.643 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:25:41.644 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:41.644 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:25:41.645 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.831934_205.1 2021-11-11T21:25:46.646 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:46.647 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:46.648 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:25:46.713 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:25:46.714 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:25:46.807 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:25:46.807 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:25:46.808 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:25:46.808 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:25:46.809 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:25:46.809 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:25:46.810 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:46.810 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:46.811 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:25:46.811 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:46.811 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:25:46.812 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:46.812 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:46.813 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:25:46.813 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:46.813 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:25:46.814 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:46.814 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:25:46.815 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:46.815 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:46.815 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:46.816 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:46.816 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:25:46.817 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:46.817 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:46.818 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:25:46.818 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:46.818 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:46.819 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:46.819 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:25:46.820 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:46.820 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:46.821 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:46.821 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:46.821 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:25:46.822 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:46.822 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:46.823 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:25:46.823 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:46.823 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:46.824 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:46.824 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:25:46.825 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:46.825 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:46.826 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:46.826 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:46.826 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:25:46.827 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:46.827 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:46.828 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:25:46.828 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:46.829 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:46.829 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:46.829 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:25:46.830 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:46.830 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:46.831 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:46.831 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:25:46.832 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:46.832 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:25:46.832 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:25:46.833 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:25:46.833 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:46.834 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:25:46.836 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.831934_205.1 2021-11-11T21:25:51.837 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:51.838 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:51.839 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:25:51.896 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:25:51.896 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:25:51.997 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:25:51.997 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:25:51.997 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:25:51.998 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:25:51.998 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:25:51.999 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:25:51.999 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:51.999 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:51.999 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:25:52.000 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:52.000 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:25:52.000 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:52.000 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:52.000 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:25:52.001 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:52.001 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:25:52.001 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:52.001 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:25:52.002 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:52.002 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:52.002 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:52.002 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:52.003 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:25:52.003 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:52.003 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:52.003 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:25:52.004 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:52.004 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:52.004 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:52.004 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:25:52.004 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:52.005 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:52.005 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:52.005 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:52.005 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:25:52.006 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:52.006 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:52.006 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:25:52.006 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:52.007 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:52.007 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:52.007 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:25:52.008 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:52.008 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:52.008 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:52.008 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:52.009 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:25:52.009 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:52.009 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:52.009 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:25:52.009 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:52.010 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:52.010 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:52.010 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:25:52.010 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:52.011 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:52.011 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:52.011 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:25:52.011 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:52.012 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:25:52.012 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:25:52.012 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:25:52.012 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:52.012 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:25:52.014 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.761008_196.1 2021-11-11T21:25:57.016 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:57.017 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:25:57.018 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:25:57.115 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:25:57.116 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:25:57.779 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:25:57.780 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:25:57.780 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:25:57.780 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:25:57.780 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:25:57.781 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:25:57.781 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:57.781 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:57.781 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:25:57.782 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:57.782 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:25:57.782 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:57.782 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:57.783 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:25:57.783 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:57.783 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:25:57.784 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:57.784 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:25:57.784 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:57.784 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:57.784 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:57.785 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:57.785 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:25:57.785 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:57.785 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:57.786 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:25:57.786 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:57.786 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:57.786 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:57.786 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:25:57.787 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:57.787 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:57.787 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:57.787 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:57.788 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:25:57.788 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:57.788 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:57.788 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:25:57.788 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:57.789 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:57.789 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:57.789 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:25:57.789 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:57.789 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:57.790 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:57.790 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:25:57.790 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:25:57.790 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:25:57.791 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:25:57.791 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:25:57.791 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:25:57.791 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:25:57.791 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:25:57.792 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:25:57.792 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:25:57.792 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:57.792 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:57.793 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:25:57.793 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:25:57.793 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:25:57.793 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:25:57.793 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:25:57.794 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:25:57.794 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:25:57.794 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.830110_204.1 2021-11-11T21:26:02.796 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:02.796 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:02.797 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:26:02.880 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:26:02.880 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:26:03.340 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:26:03.341 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:26:03.341 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:26:03.341 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:26:03.342 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:26:03.342 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:26:03.342 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:03.342 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:03.343 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:26:03.343 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:03.343 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:26:03.343 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:03.344 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:03.344 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:26:03.344 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:03.344 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:26:03.345 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:03.345 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:26:03.345 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:03.345 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:03.346 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:03.346 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:03.346 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:26:03.346 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:03.346 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:03.347 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:26:03.347 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:03.347 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:03.347 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:03.348 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:26:03.348 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:03.348 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:03.348 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:03.348 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:03.349 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:26:03.349 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:03.349 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:03.349 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:26:03.350 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:03.350 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:03.350 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:03.350 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:26:03.350 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:03.351 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:03.351 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:03.351 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:03.351 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:26:03.352 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:03.352 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:03.352 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:26:03.352 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:03.353 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:03.353 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:03.353 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:26:03.353 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:03.353 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:03.354 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:03.354 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:26:03.354 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:03.354 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:26:03.355 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:26:03.355 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:26:03.355 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:03.355 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:26:03.356 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.783136_199.1 2021-11-11T21:26:08.358 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:08.359 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:08.359 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:26:08.574 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:26:08.574 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:26:09.150 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:26:09.151 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:26:09.151 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:26:09.151 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:26:09.152 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:26:09.152 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:26:09.152 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:09.152 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:09.152 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:26:09.153 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:09.153 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:26:09.153 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:09.153 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:09.154 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:26:09.154 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:09.154 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:26:09.154 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:09.155 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:26:09.155 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:09.155 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:09.155 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:09.155 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:09.156 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:26:09.156 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:09.156 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:09.156 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:26:09.157 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:09.157 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:09.157 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:09.157 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:26:09.157 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:09.158 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:09.158 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:09.158 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:09.158 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:26:09.159 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:09.159 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:09.159 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.838825_206.1", 2021-11-11T21:26:09.159 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:09.160 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:09.160 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:09.160 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.838825Z", 2021-11-11T21:26:09.160 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:09.160 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:09.161 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:09.161 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:09.161 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:26:09.161 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:09.162 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:09.162 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:26:09.162 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:09.162 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:09.162 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:09.163 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:26:09.163 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:09.163 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:09.163 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:09.163 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:26:09.164 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:09.164 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:26:09.164 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:26:09.164 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:26:09.165 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:09.165 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:26:09.166 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.838825_206.1 2021-11-11T21:26:14.167 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:14.168 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:14.170 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:26:14.212 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:26:14.213 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:26:14.356 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:26:14.357 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:26:14.357 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:26:14.357 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:26:14.357 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:26:14.358 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:26:14.358 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:14.358 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:14.358 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:26:14.359 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:14.359 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:26:14.359 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:14.359 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:14.359 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:26:14.360 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:14.360 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:26:14.360 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:14.360 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:26:14.361 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:14.361 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:14.361 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:14.361 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:14.361 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:26:14.362 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:14.362 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:14.362 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:26:14.362 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:14.363 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:14.363 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:14.363 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:26:14.363 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:14.364 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:14.364 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:14.364 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:14.364 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:26:14.364 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:14.365 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:14.365 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:26:14.365 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:14.365 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:14.365 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:14.366 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:26:14.366 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:14.366 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:14.367 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:14.367 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:14.368 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:26:14.368 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:14.368 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:14.368 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:26:14.369 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:14.369 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:14.369 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:14.369 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:26:14.370 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:14.370 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:14.371 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:14.371 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:26:14.371 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:14.372 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:26:14.372 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:26:14.372 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:26:14.372 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:14.372 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:26:14.373 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.789745_201.1 2021-11-11T21:26:19.375 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:19.376 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:19.376 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:26:19.424 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:26:19.425 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:26:19.502 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:26:19.502 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:26:19.502 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:26:19.503 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:26:19.503 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:26:19.503 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:26:19.503 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:19.504 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:19.504 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:26:19.504 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:19.504 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:26:19.504 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:19.505 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:19.505 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:26:19.505 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:19.506 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:26:19.506 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:19.506 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:26:19.507 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:19.507 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:19.507 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:19.508 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:19.508 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:26:19.508 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:19.508 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:19.509 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:26:19.509 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:19.509 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:19.509 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:19.509 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:26:19.510 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:19.510 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:19.510 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:19.510 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:19.511 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:26:19.511 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:19.511 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:19.511 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:26:19.512 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:19.512 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:19.512 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:19.512 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:26:19.513 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:19.513 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:19.513 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:19.513 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:19.513 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:26:19.514 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:19.514 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:19.514 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:26:19.514 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:19.515 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:19.515 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:19.515 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:26:19.515 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:19.516 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:19.516 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:19.516 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:26:19.516 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:19.517 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:26:19.517 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:26:19.518 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:26:19.518 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:19.518 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:26:19.520 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:26:24.521 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:24.522 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:24.523 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:26:24.566 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:26:24.567 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:26:24.636 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:26:24.637 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:26:24.637 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:26:24.637 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:26:24.637 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:26:24.638 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:26:24.638 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:24.638 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:24.638 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:26:24.639 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:24.639 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:26:24.639 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:24.639 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:24.639 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:26:24.640 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:24.640 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:26:24.640 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:24.640 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:26:24.641 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:24.641 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:24.641 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:24.641 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:24.642 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:26:24.642 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:24.642 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:24.642 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:26:24.642 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:24.643 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:24.643 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:24.643 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:26:24.643 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:24.644 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:24.644 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:24.644 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:24.644 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:26:24.644 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:24.645 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:24.645 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:26:24.645 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:24.645 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:24.646 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:24.646 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:26:24.646 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:24.646 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:24.647 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:24.647 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:24.647 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:26:24.647 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:24.647 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:24.648 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:26:24.648 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:24.648 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:24.649 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:24.649 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:26:24.649 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:24.649 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:24.650 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:24.650 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:26:24.650 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:24.650 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:26:24.651 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:26:24.651 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:26:24.651 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:24.651 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:26:24.653 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.789745_201.1 2021-11-11T21:26:29.654 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:29.655 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:29.655 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:26:29.707 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:26:29.707 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:26:30.108 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:26:30.108 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:26:30.109 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:26:30.109 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:26:30.109 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:26:30.109 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:26:30.109 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:30.110 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:30.110 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:26:30.110 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:30.110 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:26:30.111 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:30.111 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:30.111 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:26:30.111 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:30.112 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:26:30.112 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:30.112 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:26:30.113 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:30.113 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:30.113 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:30.113 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:30.114 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:26:30.114 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:30.114 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:30.115 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:26:30.115 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:30.115 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:30.115 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:30.116 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:26:30.116 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:30.116 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:30.117 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:30.117 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:30.117 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:26:30.117 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:30.118 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:30.118 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:26:30.118 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:30.118 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:30.119 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:30.119 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:26:30.119 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:30.119 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:30.120 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:30.120 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:30.120 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:26:30.120 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:30.121 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:30.121 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:26:30.121 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:30.122 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:30.123 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:30.123 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:26:30.123 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:30.123 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:30.124 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:30.124 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:26:30.124 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:30.124 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:26:30.125 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:26:30.125 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:26:30.125 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:30.126 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:26:30.126 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.825641_203.1 2021-11-11T21:26:35.128 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:35.128 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:35.129 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:26:35.172 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:26:35.172 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:26:35.250 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:26:35.250 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:26:35.250 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:26:35.251 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:26:35.251 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:26:35.251 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:26:35.251 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:35.252 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:35.252 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:26:35.252 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:35.252 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:26:35.253 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:35.253 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:35.253 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:26:35.253 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:35.253 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:26:35.254 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:35.254 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:26:35.254 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:35.254 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:35.255 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:35.255 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:35.255 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:26:35.255 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:35.255 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:35.256 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:26:35.256 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:35.256 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:35.256 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:35.257 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:26:35.257 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:35.257 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:35.257 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:35.257 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:35.258 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:26:35.258 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:35.258 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:35.258 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:26:35.259 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:35.259 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:35.259 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:35.259 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:26:35.259 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:35.260 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:35.260 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:35.260 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:35.260 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:26:35.261 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:35.261 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:35.261 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:26:35.261 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:35.261 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:35.262 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:35.262 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:26:35.262 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:35.262 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:35.263 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:35.263 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:26:35.263 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:35.263 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:26:35.263 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:26:35.264 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:26:35.264 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:35.264 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:26:35.265 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665225.075927_221.1 2021-11-11T21:26:40.266 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:40.267 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:40.268 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:26:40.313 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:26:40.313 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:26:40.405 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:26:40.405 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:26:40.405 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:26:40.405 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:26:40.406 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:26:40.406 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:26:40.406 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:40.406 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:40.407 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:26:40.407 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:40.407 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:26:40.407 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:40.408 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:40.408 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:26:40.408 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:40.408 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:26:40.408 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:40.409 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:26:40.409 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:40.409 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:40.409 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:40.410 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:40.410 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:26:40.410 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:40.410 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:40.410 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:26:40.411 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:40.411 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:40.411 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:40.411 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:26:40.412 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:40.412 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:40.412 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:40.412 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:40.412 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:26:40.413 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:40.413 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:40.413 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:26:40.413 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:40.414 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:40.414 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:40.414 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:26:40.414 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:40.414 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:40.415 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:40.415 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:40.415 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:26:40.415 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:40.416 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:40.416 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:26:40.416 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:40.416 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:40.416 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:40.417 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:26:40.417 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:40.417 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:40.417 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:40.418 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:26:40.418 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:40.418 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:26:40.418 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:26:40.418 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:26:40.419 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:40.419 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:26:40.420 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.787834_200.1 2021-11-11T21:26:45.422 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:45.423 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:45.424 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:26:45.468 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:26:45.469 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:26:45.551 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:26:45.551 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:26:45.552 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:26:45.552 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:26:45.552 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:26:45.552 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:26:45.553 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:45.553 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:45.553 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:26:45.554 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:45.554 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:26:45.554 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:45.554 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:45.555 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:26:45.555 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:45.555 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:26:45.555 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:45.556 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:26:45.556 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:45.556 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:45.556 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:45.557 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:45.557 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:26:45.557 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:45.557 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:45.557 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:26:45.558 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:45.558 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:45.558 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:45.558 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:26:45.559 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:45.559 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:45.559 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:45.559 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:45.560 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:26:45.560 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:45.560 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:45.560 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:26:45.560 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:45.561 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:45.561 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:45.561 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:26:45.561 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:45.562 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:45.562 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:45.562 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:45.562 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:26:45.562 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:45.563 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:45.563 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:26:45.563 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:45.563 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:45.564 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:45.564 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:26:45.564 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:45.564 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:45.565 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:45.565 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:26:45.565 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:45.565 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:26:45.566 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:26:45.566 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:26:45.566 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:45.566 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:26:45.568 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.831934_205.1 2021-11-11T21:26:50.569 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:50.570 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:50.571 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:26:50.657 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:26:50.657 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:26:50.921 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:26:50.921 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:26:50.921 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:26:50.921 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:26:50.922 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:26:50.922 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:26:50.922 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:50.922 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:50.923 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:26:50.923 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:50.923 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:26:50.923 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:50.924 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:50.924 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:26:50.924 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:50.924 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:26:50.925 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:50.925 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:26:50.925 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:50.925 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:50.925 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:50.926 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:50.926 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:26:50.926 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:50.926 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:50.927 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:26:50.927 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:50.927 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:50.927 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:50.928 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:26:50.928 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:50.928 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:50.928 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:50.928 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:50.929 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:26:50.929 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:50.929 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:50.929 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:26:50.930 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:50.930 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:50.930 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:50.930 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:26:50.930 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:50.931 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:50.931 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:50.931 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:50.931 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:26:50.932 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:50.932 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:50.932 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:26:50.932 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:50.932 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:50.933 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:50.933 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:26:50.933 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:50.933 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:50.934 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:50.934 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:26:50.934 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:50.934 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:26:50.935 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:26:50.935 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:26:50.935 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:50.935 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:26:50.936 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:26:55.937 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:55.938 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:26:55.938 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:26:56.008 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:26:56.009 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:26:56.123 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:26:56.124 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:26:56.124 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:26:56.124 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:26:56.124 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:26:56.125 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:26:56.125 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:56.125 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:56.125 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:26:56.126 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:56.126 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:26:56.126 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:56.126 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:56.127 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:26:56.127 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:56.127 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:26:56.127 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:56.128 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:26:56.128 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:56.128 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:56.128 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:56.129 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:56.129 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:26:56.129 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:56.129 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:56.130 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:26:56.130 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:56.130 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:56.130 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:56.131 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:26:56.131 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:56.131 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:56.131 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:56.131 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:56.132 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:26:56.132 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:56.132 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:56.132 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:26:56.133 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:56.133 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:56.133 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:56.133 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:26:56.134 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:56.134 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:56.134 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:56.134 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:26:56.134 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:26:56.135 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:26:56.135 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:26:56.135 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:26:56.135 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:26:56.136 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:26:56.136 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:26:56.136 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:26:56.136 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:26:56.137 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:56.137 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:56.137 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:26:56.137 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:26:56.137 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:26:56.138 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:26:56.138 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:26:56.138 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:26:56.138 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:26:56.140 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.789745_201.1 2021-11-11T21:27:01.142 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:01.143 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:01.144 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:27:01.338 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:27:01.338 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:27:01.873 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:27:01.874 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:27:01.874 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:27:01.874 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:27:01.875 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:27:01.875 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:27:01.875 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:01.875 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:01.876 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:27:01.876 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:01.876 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:27:01.876 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:01.877 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:01.877 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:27:01.877 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:01.877 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:27:01.877 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:01.878 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:27:01.878 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:01.878 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:01.878 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:01.879 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:01.879 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:27:01.879 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:01.879 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:01.880 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:27:01.880 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:01.880 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:01.881 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:01.881 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:27:01.881 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:01.881 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:01.882 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:01.882 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:01.882 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:27:01.883 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:01.883 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:01.883 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:27:01.883 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:01.883 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:01.884 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:01.884 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:27:01.884 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:01.884 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:01.885 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:01.885 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:01.885 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:27:01.885 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:01.885 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:01.886 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:27:01.886 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:01.886 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:01.886 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:01.887 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:27:01.887 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:01.887 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:01.887 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:01.887 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:27:01.888 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:01.888 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:27:01.888 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:27:01.888 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:27:01.889 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:01.889 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:27:01.890 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:27:06.891 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:06.892 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:06.893 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:27:07.067 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:27:07.068 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:27:07.146 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:27:07.146 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:27:07.146 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:27:07.147 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:27:07.147 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:27:07.147 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:27:07.147 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:07.148 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:07.148 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:27:07.148 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:07.148 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:27:07.149 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:07.149 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:07.149 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:27:07.149 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:07.149 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:27:07.150 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:07.150 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:27:07.150 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:07.150 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:07.151 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:07.151 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:07.151 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:27:07.151 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:07.151 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:07.152 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:27:07.152 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:07.152 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:07.152 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:07.153 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:27:07.153 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:07.153 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:07.153 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:07.153 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:07.154 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:27:07.154 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:07.154 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:07.154 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:27:07.155 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:07.155 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:07.155 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:07.155 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:27:07.156 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:07.156 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:07.156 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:07.156 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:07.157 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:27:07.157 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:07.157 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:07.157 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:27:07.157 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:07.158 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:07.158 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:07.158 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:27:07.158 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:07.158 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:07.159 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:07.159 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:27:07.159 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:07.159 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:27:07.160 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:27:07.160 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:27:07.160 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:07.160 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:27:07.161 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:27:12.163 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:12.163 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:12.164 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:27:12.207 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:27:12.208 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:27:12.276 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:27:12.276 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:27:12.276 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:27:12.277 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:27:12.277 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:27:12.277 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:27:12.277 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:12.278 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:12.278 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:27:12.278 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:12.278 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:27:12.279 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:12.279 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:12.279 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:27:12.279 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:12.279 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:27:12.280 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:12.280 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:27:12.280 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:12.280 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:12.281 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:12.281 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:12.281 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:27:12.281 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:12.281 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:12.282 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:27:12.282 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:12.282 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:12.282 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:12.283 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:27:12.283 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:12.283 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:12.283 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:12.283 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:12.284 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:27:12.284 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:12.284 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:12.284 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:27:12.285 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:12.285 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:12.285 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:12.285 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:27:12.285 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:12.286 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:12.286 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:12.286 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:12.286 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:27:12.287 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:12.287 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:12.287 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:27:12.287 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:12.288 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:12.288 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:12.288 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:27:12.288 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:12.288 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:12.289 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:12.289 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:27:12.289 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:12.289 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:27:12.290 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:27:12.290 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:27:12.290 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:12.290 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:27:12.291 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.768842_198.1 2021-11-11T21:27:17.293 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:17.293 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:17.294 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:27:17.392 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:27:17.393 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:27:17.683 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:27:17.684 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:27:17.685 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:27:17.685 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:27:17.685 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:27:17.686 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:27:17.686 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:17.686 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:17.686 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:27:17.686 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:17.687 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:27:17.687 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:17.687 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:17.688 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:27:17.688 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:17.688 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:27:17.688 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:17.689 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:27:17.689 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:17.689 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:17.689 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:17.689 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:17.690 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:27:17.690 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:17.690 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:17.690 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:27:17.691 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:17.691 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:17.691 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:17.692 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:27:17.692 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:17.692 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:17.692 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:17.693 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:17.693 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:27:17.693 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:17.693 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:17.694 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:27:17.694 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:17.694 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:17.694 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:17.695 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:27:17.695 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:17.695 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:17.695 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:17.696 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:17.696 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:27:17.696 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:17.696 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:17.696 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:27:17.697 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:17.697 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:17.697 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:17.697 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:27:17.698 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:17.698 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:17.698 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:17.698 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:27:17.699 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:17.699 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:27:17.699 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:27:17.699 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:27:17.699 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:17.700 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:27:17.701 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:27:22.702 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:22.703 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:22.703 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:27:22.757 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:27:22.757 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:27:22.859 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:27:22.860 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:27:22.860 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:27:22.860 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:27:22.860 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:27:22.860 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:27:22.861 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:22.861 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:22.861 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:27:22.861 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:22.862 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:27:22.862 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:22.862 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:22.862 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:27:22.863 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:22.863 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:27:22.863 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:22.863 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:27:22.863 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:22.864 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:22.864 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:22.864 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:22.864 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:27:22.865 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:22.865 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:22.865 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:27:22.865 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:22.866 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:22.866 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:22.866 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:27:22.866 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:22.866 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:22.867 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:22.867 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:22.867 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:27:22.867 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:22.868 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:22.868 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:27:22.868 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:22.868 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:22.868 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:22.869 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:27:22.869 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:22.869 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:22.869 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:22.870 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:22.870 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:27:22.870 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:22.870 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:22.870 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:27:22.871 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:22.871 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:22.871 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:22.871 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:27:22.872 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:22.872 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:22.872 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:22.873 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:27:22.873 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:22.873 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:27:22.873 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:27:22.874 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:27:22.874 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:22.874 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:27:22.875 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.761008_196.1 2021-11-11T21:27:27.876 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:27.877 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:27.878 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:27:27.927 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:27:27.928 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:27:28.018 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:27:28.019 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:27:28.019 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:27:28.019 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:27:28.019 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:27:28.020 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:27:28.020 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:28.020 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:28.020 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:27:28.021 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:28.021 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:27:28.021 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:28.021 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:28.022 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:27:28.022 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:28.022 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:27:28.022 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:28.023 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:27:28.023 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:28.024 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:28.024 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:28.024 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:28.024 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:27:28.024 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:28.025 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:28.025 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:27:28.025 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:28.025 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:28.025 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:28.026 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:27:28.026 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:28.026 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:28.026 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:28.027 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:28.027 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:27:28.027 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:28.027 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:28.027 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:27:28.028 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:28.028 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:28.028 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:28.028 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:27:28.028 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:28.029 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:28.029 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:28.029 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:28.030 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:27:28.030 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:28.030 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:28.030 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:27:28.031 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:28.031 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:28.031 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:28.031 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:27:28.031 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:28.032 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:28.032 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:28.032 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:27:28.032 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:28.032 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:27:28.033 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:27:28.033 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:27:28.033 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:28.033 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:27:28.035 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:27:33.036 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:33.037 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:33.038 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:27:33.080 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:27:33.081 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:27:33.155 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:27:33.156 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:27:33.156 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:27:33.156 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:27:33.157 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:27:33.157 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:27:33.157 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:33.157 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:33.158 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:27:33.158 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:33.158 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:27:33.158 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:33.159 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:33.159 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:27:33.159 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:33.159 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:27:33.160 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:33.160 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:27:33.160 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:33.160 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:33.160 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:33.161 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:33.161 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:27:33.161 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:33.161 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:33.162 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:27:33.162 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:33.162 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:33.162 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:33.162 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:27:33.163 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:33.163 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:33.163 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:33.163 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:33.164 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:27:33.164 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:33.164 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:33.164 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:27:33.164 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:33.165 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:33.165 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:33.165 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:27:33.165 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:33.166 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:33.166 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:33.166 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:33.166 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:27:33.167 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:33.167 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:33.167 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:27:33.167 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:33.167 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:33.168 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:33.168 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:27:33.168 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:33.168 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:33.169 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:33.169 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:27:33.169 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:33.169 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:27:33.169 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:27:33.170 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:27:33.170 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:33.170 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:27:33.171 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.830110_204.1 2021-11-11T21:27:38.172 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:38.173 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:38.174 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:27:38.220 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:27:38.221 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:27:38.590 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:27:38.590 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:27:38.590 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:27:38.591 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:27:38.591 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:27:38.591 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:27:38.591 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:38.592 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:38.592 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:27:38.592 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:38.593 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:27:38.593 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:38.593 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:38.593 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:27:38.594 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:38.594 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:27:38.594 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:38.594 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:27:38.595 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:38.595 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:38.595 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:38.595 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:38.596 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:27:38.597 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:38.597 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:38.597 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:27:38.598 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:38.598 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:38.598 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:38.598 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:27:38.599 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:38.599 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:38.599 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:38.599 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:38.599 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:27:38.600 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:38.600 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:38.600 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:27:38.600 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:38.601 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:38.601 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:38.601 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:27:38.601 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:38.601 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:38.602 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:38.602 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:38.602 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:27:38.602 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:38.603 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:38.603 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:27:38.603 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:38.603 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:38.604 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:38.604 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:27:38.604 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:38.604 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:38.604 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:38.605 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:27:38.605 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:38.605 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:27:38.605 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:27:38.605 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:27:38.606 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:38.606 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:27:38.607 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.789745_201.1 2021-11-11T21:27:43.609 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:43.610 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:43.610 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:27:43.654 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:27:43.654 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:27:43.756 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:27:43.757 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:27:43.757 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:27:43.758 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:27:43.758 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:27:43.759 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:27:43.759 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:43.759 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:43.760 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:27:43.760 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:43.760 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:27:43.760 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:43.760 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:43.761 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:27:43.761 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:43.761 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:27:43.761 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:43.761 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:27:43.762 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:43.762 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:43.762 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:43.762 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:43.763 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:27:43.763 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:43.763 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:43.763 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:27:43.763 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:43.764 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:43.764 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:43.764 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:27:43.764 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:43.765 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:43.765 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:43.765 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:43.765 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:27:43.766 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:43.766 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:43.766 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.050883_217.1", 2021-11-11T21:27:43.766 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:43.767 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:43.767 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:43.767 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.050883Z", 2021-11-11T21:27:43.767 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:43.768 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:43.768 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:43.768 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:43.768 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:27:43.768 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:43.769 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:43.769 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:27:43.769 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:43.769 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:43.770 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:43.770 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:27:43.770 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:43.770 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:43.771 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:43.771 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:27:43.771 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:43.771 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:27:43.771 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:27:43.772 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:27:43.772 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:43.772 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:27:43.773 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665225.050883_217.1 2021-11-11T21:27:48.775 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:48.776 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:48.776 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:27:48.858 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:27:48.859 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:27:49.424 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:27:49.425 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:27:49.425 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:27:49.425 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:27:49.426 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:27:49.426 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:27:49.426 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:49.426 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:49.426 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:27:49.427 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:49.427 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:27:49.427 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:49.427 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:49.428 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:27:49.428 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:49.428 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:27:49.428 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:49.429 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:27:49.429 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:49.429 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:49.429 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:49.430 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:49.430 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:27:49.430 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:49.430 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:49.430 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:27:49.431 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:49.431 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:49.431 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:49.431 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:27:49.432 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:49.432 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:49.432 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:49.432 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:49.432 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:27:49.433 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:49.433 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:49.433 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:27:49.433 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:49.434 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:49.434 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:49.434 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:27:49.434 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:49.435 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:49.435 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:49.435 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:49.435 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:27:49.435 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:49.436 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:49.436 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:27:49.436 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:49.436 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:49.437 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:49.437 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:27:49.437 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:49.437 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:49.437 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:49.438 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:27:49.438 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:49.438 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:27:49.438 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:27:49.439 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:27:49.439 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:49.439 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:27:49.440 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.825641_203.1 2021-11-11T21:27:54.442 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:54.442 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:27:54.443 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:27:54.514 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:27:54.514 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:27:55.084 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:27:55.085 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:27:55.085 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:27:55.085 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:27:55.085 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:27:55.086 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:27:55.086 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:55.086 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:55.086 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:27:55.087 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:55.087 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:27:55.087 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:55.087 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:55.087 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:27:55.088 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:55.088 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:27:55.088 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:55.088 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:27:55.088 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:55.089 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:55.089 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:55.089 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:55.089 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:27:55.089 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:55.090 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:55.090 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:27:55.090 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:55.090 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:55.091 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:55.091 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:27:55.091 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:55.091 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:55.092 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:55.092 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:55.092 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:27:55.092 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:55.092 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:55.093 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:27:55.093 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:55.093 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:55.093 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:55.094 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:27:55.094 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:55.094 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:55.094 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:55.094 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:27:55.095 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:27:55.095 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:27:55.095 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:27:55.095 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:27:55.095 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:27:55.096 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:27:55.096 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:27:55.096 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:27:55.096 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:27:55.097 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:55.097 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:55.097 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:27:55.097 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:27:55.098 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:27:55.098 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:27:55.098 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:27:55.098 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:27:55.099 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:27:55.099 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:28:00.100 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:00.101 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:00.101 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:00.150 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:00.150 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:00.236 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:00.236 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:00.236 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:00.236 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:00.237 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:00.237 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:00.237 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:00.238 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:00.238 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:00.238 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:00.238 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:00.239 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:00.239 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:00.239 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:00.239 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:00.239 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:00.240 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:00.240 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:00.240 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:00.240 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:00.240 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:00.241 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:00.241 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:00.241 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:00.241 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:00.242 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:00.242 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:00.242 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:00.242 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:00.243 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:00.243 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:00.243 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:00.243 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:00.243 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:00.244 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:00.244 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:00.244 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:00.245 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.838825_206.1", 2021-11-11T21:28:00.245 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:00.245 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:00.245 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:00.245 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.838825Z", 2021-11-11T21:28:00.246 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:00.246 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:00.246 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:00.247 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:00.247 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:00.247 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:00.248 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:00.248 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:00.248 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:00.249 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:00.249 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:00.250 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:00.250 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:00.250 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:00.251 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:00.251 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:00.252 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:00.252 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:00.252 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:00.253 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:00.253 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:00.254 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:00.256 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.838825_206.1 2021-11-11T21:28:05.258 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:05.259 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:05.259 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:05.306 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:05.307 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:05.383 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:05.383 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:05.384 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:05.384 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:05.384 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:05.384 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:05.385 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:05.385 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:05.385 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:05.385 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:05.385 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:05.386 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:05.386 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:05.386 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:05.386 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:05.387 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:05.387 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:05.387 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:05.387 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:05.388 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:05.388 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:05.388 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:05.388 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:05.388 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:05.389 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:05.389 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:05.389 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:05.389 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:05.390 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:05.390 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:05.390 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:05.390 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:05.390 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:05.391 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:05.391 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:05.391 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:05.392 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:05.392 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.838825_206.1", 2021-11-11T21:28:05.392 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:05.392 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:05.393 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:05.393 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.838825Z", 2021-11-11T21:28:05.393 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:05.393 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:05.393 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:05.394 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:05.394 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:05.394 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:05.394 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:05.395 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:05.395 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:05.395 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:05.395 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:05.396 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:05.396 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:05.396 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:05.396 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:05.396 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:05.397 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:05.397 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:05.397 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:05.397 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:05.398 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:05.398 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:05.399 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.838825_206.1 2021-11-11T21:28:10.401 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:10.402 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:10.402 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:10.481 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:10.482 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:10.909 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:10.910 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:10.910 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:10.910 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:10.910 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:10.911 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:10.911 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:10.911 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:10.911 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:10.912 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:10.912 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:10.912 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:10.912 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:10.913 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:10.913 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:10.913 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:10.913 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:10.913 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:10.914 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:10.914 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:10.914 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:10.914 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:10.915 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:10.915 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:10.915 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:10.915 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:10.915 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:10.916 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:10.916 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:10.916 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:10.916 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:10.917 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:10.917 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:10.917 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:10.917 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:10.917 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:10.918 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:10.918 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:28:10.918 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:10.918 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:10.919 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:10.919 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:28:10.919 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:10.919 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:10.919 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:10.920 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:10.920 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:10.920 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:10.920 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:10.921 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:10.921 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:10.921 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:10.921 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:10.921 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:10.922 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:10.922 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:10.922 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:10.922 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:10.923 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:10.923 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:10.923 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:10.923 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:10.924 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:10.924 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:10.925 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.789745_201.1 2021-11-11T21:28:15.927 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:15.928 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:15.928 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:16.007 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:16.007 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:16.154 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:16.154 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:16.155 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:16.155 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:16.155 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:16.155 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:16.155 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:16.156 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:16.156 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:16.156 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:16.156 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:16.157 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:16.157 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:16.157 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:16.157 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:16.157 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:16.158 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:16.158 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:16.158 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:16.158 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:16.159 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:16.159 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:16.159 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:16.159 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:16.159 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:16.160 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:16.160 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:16.160 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:16.160 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:16.161 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:16.161 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:16.161 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:16.161 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:16.161 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:16.162 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:16.162 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:16.162 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:16.162 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.838825_206.1", 2021-11-11T21:28:16.163 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:16.163 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:16.163 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:16.163 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.838825Z", 2021-11-11T21:28:16.163 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:16.164 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:16.164 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:16.164 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:16.164 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:16.165 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:16.165 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:16.165 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:16.165 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:16.166 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:16.166 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:16.166 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:16.166 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:16.167 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:16.167 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:16.167 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:16.167 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:16.167 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:16.168 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:16.168 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:16.168 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:16.168 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:16.169 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.838825_206.1 2021-11-11T21:28:21.171 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:21.171 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:21.172 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:21.218 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:21.219 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:21.589 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:21.589 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:21.589 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:21.589 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:21.590 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:21.590 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:21.590 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:21.590 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:21.591 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:21.591 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:21.591 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:21.591 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:21.592 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:21.592 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:21.592 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:21.592 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:21.593 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:21.593 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:21.593 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:21.593 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:21.593 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:21.594 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:21.594 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:21.594 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:21.594 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:21.595 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:21.595 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:21.595 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:21.595 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:21.596 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:21.596 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:21.596 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:21.596 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:21.596 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:21.597 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:21.597 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:21.597 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:21.597 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.838825_206.1", 2021-11-11T21:28:21.598 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:21.598 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:21.598 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:21.598 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.838825Z", 2021-11-11T21:28:21.599 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:21.599 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:21.599 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:21.599 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:21.599 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:21.600 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:21.600 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:21.600 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:21.600 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:21.600 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:21.601 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:21.601 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:21.601 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:21.601 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:21.602 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:21.602 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:21.602 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:21.602 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:21.603 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:21.603 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:21.603 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:21.603 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:21.604 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.838825_206.1 2021-11-11T21:28:26.605 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:26.606 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:26.607 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:26.655 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:26.656 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:26.735 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:26.735 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:26.735 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:26.736 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:26.736 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:26.736 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:26.736 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:26.736 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:26.737 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:26.737 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:26.737 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:26.737 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:26.737 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:26.738 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:26.738 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:26.738 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:26.738 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:26.739 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:26.739 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:26.739 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:26.739 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:26.739 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:26.740 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:26.740 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:26.740 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:26.740 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:26.741 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:26.741 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:26.741 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:26.741 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:26.741 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:26.742 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:26.742 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:26.742 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:26.742 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:26.742 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:26.743 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:26.743 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:28:26.743 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:26.743 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:26.743 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:26.744 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:28:26.744 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:26.744 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:26.744 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:26.744 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:26.745 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:26.745 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:26.745 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:26.745 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:26.745 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:26.746 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:26.746 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:26.746 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:26.746 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:26.747 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:26.747 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:26.747 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:26.747 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:26.747 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:26.748 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:26.748 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:26.748 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:26.748 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:26.749 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.831934_205.1 2021-11-11T21:28:31.750 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:31.751 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:31.752 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:31.831 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:31.832 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:31.948 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:31.948 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:31.948 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:31.949 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:31.949 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:31.949 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:31.949 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:31.949 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:31.950 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:31.950 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:31.950 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:31.950 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:31.951 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:31.951 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:31.951 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:31.951 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:31.951 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:31.952 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:31.952 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:31.952 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:31.952 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:31.952 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:31.953 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:31.953 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:31.953 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:31.953 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:31.954 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:31.954 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:31.954 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:31.954 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:31.954 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:31.955 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:31.955 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:31.955 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:31.955 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:31.956 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:31.956 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:31.956 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:28:31.956 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:31.957 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:31.957 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:31.957 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:28:31.957 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:31.957 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:31.958 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:31.958 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:31.958 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:31.958 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:31.958 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:31.959 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:31.959 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:31.959 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:31.959 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:31.960 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:31.960 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:31.960 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:31.960 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:31.960 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:31.961 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:31.961 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:31.961 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:31.961 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:31.961 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:31.962 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:31.963 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:28:36.964 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:36.965 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:36.965 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:37.044 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:37.044 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:37.118 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:37.118 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:37.118 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:37.119 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:37.119 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:37.119 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:37.119 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:37.120 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:37.120 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:37.120 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:37.120 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:37.121 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:37.121 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:37.121 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:37.121 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:37.122 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:37.122 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:37.122 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:37.122 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:37.122 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:37.123 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:37.123 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:37.123 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:37.123 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:37.124 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:37.124 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:37.124 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:37.124 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:37.124 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:37.125 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:37.125 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:37.125 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:37.125 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:37.125 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:37.126 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:37.126 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:37.126 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:37.126 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:28:37.126 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:37.127 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:37.127 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:37.127 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:28:37.127 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:37.127 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:37.128 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:37.128 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:37.128 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:37.128 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:37.129 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:37.129 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:37.129 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:37.129 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:37.129 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:37.130 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:37.130 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:37.130 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:37.130 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:37.130 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:37.131 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:37.131 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:37.131 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:37.131 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:37.131 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:37.132 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:37.133 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.761008_196.1 2021-11-11T21:28:42.134 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:42.243 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:42.244 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:42.296 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:42.297 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:42.451 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:42.452 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:42.452 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:42.452 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:42.453 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:42.453 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:42.453 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:42.453 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:42.454 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:42.454 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:42.454 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:42.454 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:42.455 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:42.455 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:42.455 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:42.455 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:42.455 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:42.456 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:42.456 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:42.456 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:42.456 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:42.457 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:42.457 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:42.457 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:42.457 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:42.457 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:42.458 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:42.458 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:42.458 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:42.459 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:42.459 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:42.459 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:42.459 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:42.459 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:42.460 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:42.460 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:42.460 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:42.460 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.050883_217.1", 2021-11-11T21:28:42.461 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:42.461 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:42.462 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:42.462 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.050883Z", 2021-11-11T21:28:42.462 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:42.462 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:42.463 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:42.463 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:42.464 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:42.464 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:42.464 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:42.465 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:42.465 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:42.465 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:42.465 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:42.466 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:42.466 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:42.466 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:42.467 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:42.467 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:42.467 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:42.467 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:42.467 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:42.468 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:42.468 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:42.468 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:42.470 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665225.050883_217.1 2021-11-11T21:28:47.471 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:47.471 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:47.472 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:47.519 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:47.520 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:47.591 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:47.591 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:47.591 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:47.592 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:47.592 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:47.592 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:47.592 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:47.593 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:47.593 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:47.593 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:47.593 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:47.594 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:47.594 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:47.594 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:47.594 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:47.595 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:47.595 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:47.595 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:47.595 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:47.595 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:47.596 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:47.596 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:47.596 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:47.596 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:47.597 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:47.597 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:47.597 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:47.597 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:47.597 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:47.598 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:47.598 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:47.598 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:47.598 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:47.599 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:47.599 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:47.599 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:47.599 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:47.600 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.074615_220.1", 2021-11-11T21:28:47.600 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:47.600 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:47.600 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:47.601 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.074615Z", 2021-11-11T21:28:47.601 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:47.601 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:47.601 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:47.601 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:47.602 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:47.602 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:47.602 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:47.602 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:47.603 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:47.603 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:47.603 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:47.603 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:47.603 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:47.604 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:47.604 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:47.605 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:47.605 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:47.606 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:47.606 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:47.606 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:47.606 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:47.607 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:47.608 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665225.074615_220.1 2021-11-11T21:28:52.609 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:52.610 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:52.611 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:52.664 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:52.664 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:53.124 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:53.125 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:53.125 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:53.125 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:53.125 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:53.126 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:53.126 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:53.126 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:53.126 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:53.127 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:53.127 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:53.127 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:53.127 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:53.128 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:53.128 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:53.128 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:53.128 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:53.129 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:53.129 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:53.129 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:53.129 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:53.129 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:53.130 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:53.130 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:53.130 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:53.130 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:53.131 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:53.131 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:53.131 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:53.131 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:53.132 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:53.132 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:53.132 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:53.132 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:53.133 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:53.133 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:53.133 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:53.133 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.761008_196.1", 2021-11-11T21:28:53.133 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:53.134 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:53.134 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:53.134 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.761008Z", 2021-11-11T21:28:53.134 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:53.135 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:53.135 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:53.135 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:53.135 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:53.136 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:53.136 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:53.136 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:53.136 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:53.137 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:53.137 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:53.137 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:53.137 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:53.137 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:53.138 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:53.138 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:53.138 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:53.138 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:53.139 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:53.139 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:53.139 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:53.139 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:53.141 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.761008_196.1 2021-11-11T21:28:58.142 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:58.143 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:28:58.144 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:28:58.224 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:28:58.224 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:28:58.559 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:28:58.559 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:28:58.560 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:28:58.560 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:28:58.560 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:28:58.560 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:28:58.561 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:58.561 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:58.561 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:28:58.561 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:58.562 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:28:58.562 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:58.562 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:58.562 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:28:58.563 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:58.564 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:28:58.564 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:58.564 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:28:58.564 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:58.565 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:58.565 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:58.565 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:58.565 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:28:58.566 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:58.566 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:58.566 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:28:58.566 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:58.567 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:58.567 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:58.567 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:28:58.567 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:58.568 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:58.568 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:58.568 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:58.568 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:28:58.568 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:58.569 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:58.569 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:28:58.569 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:58.570 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:58.570 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:58.570 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:28:58.570 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:58.571 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:58.571 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:58.571 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:28:58.571 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:28:58.571 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:28:58.572 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:28:58.572 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:28:58.572 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:28:58.572 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:28:58.573 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:28:58.573 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:28:58.573 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:28:58.574 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:58.574 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:58.574 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:28:58.574 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:28:58.574 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:28:58.575 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:28:58.575 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:28:58.575 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:28:58.575 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:28:58.577 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:29:03.578 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:03.579 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:03.579 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:29:03.646 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:29:03.646 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:29:04.231 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:29:04.231 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:29:04.232 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:29:04.232 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:29:04.232 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:29:04.232 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:29:04.233 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:04.233 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:04.233 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:29:04.233 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:04.233 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:29:04.234 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:04.234 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:04.234 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:29:04.234 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:04.235 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:29:04.235 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:04.235 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:29:04.235 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:04.236 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:04.236 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:04.236 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:04.236 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:29:04.236 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:04.237 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:04.237 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:29:04.237 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:04.237 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:04.237 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:04.238 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:29:04.238 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:04.238 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:04.238 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:04.239 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:04.239 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:29:04.239 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:04.239 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:04.240 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.044532_215.1", 2021-11-11T21:29:04.240 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:04.240 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:04.240 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:04.240 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.044532Z", 2021-11-11T21:29:04.241 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:04.241 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:04.241 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:04.241 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:04.241 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:29:04.242 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:04.242 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:04.242 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:29:04.242 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:04.242 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:04.243 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:04.243 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:29:04.243 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:04.243 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:04.243 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:04.244 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:29:04.244 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:04.244 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:29:04.244 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:29:04.244 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:29:04.245 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:04.245 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:29:04.246 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665225.044532_215.1 2021-11-11T21:29:09.248 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:09.248 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:09.249 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:29:09.341 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:29:09.341 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:29:09.895 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:29:09.895 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:29:09.895 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:29:09.896 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:29:09.896 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:29:09.896 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:29:09.896 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:09.897 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:09.897 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:29:09.897 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:09.897 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:29:09.898 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:09.898 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:09.898 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:29:09.898 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:09.898 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:29:09.899 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:09.899 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:29:09.899 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:09.899 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:09.900 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:09.900 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:09.900 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:29:09.900 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:09.900 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:09.901 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:29:09.901 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:09.901 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:09.901 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:09.902 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:29:09.902 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:09.902 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:09.902 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:09.903 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:09.903 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:29:09.903 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:09.903 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:09.903 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:29:09.904 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:09.904 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:09.904 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:09.904 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:29:09.905 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:09.905 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:09.905 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:09.905 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:09.905 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:29:09.906 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:09.906 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:09.906 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:29:09.906 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:09.907 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:09.907 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:09.907 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:29:09.907 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:09.907 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:09.908 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:09.908 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:29:09.908 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:09.908 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:29:09.909 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:29:09.909 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:29:09.909 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:09.909 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:29:09.911 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.789745_201.1 2021-11-11T21:29:14.912 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:14.913 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:14.913 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:29:14.963 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:29:14.963 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:29:15.036 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:29:15.036 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:29:15.037 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:29:15.037 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:29:15.037 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:29:15.038 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:29:15.038 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:15.038 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:15.038 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:29:15.039 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:15.039 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:29:15.039 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:15.039 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:15.040 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:29:15.040 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:15.040 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:29:15.040 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:15.040 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:29:15.041 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:15.041 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:15.041 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:15.041 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:15.042 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:29:15.042 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:15.042 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:15.042 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:29:15.043 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:15.043 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:15.043 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:15.043 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:29:15.044 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:15.044 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:15.044 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:15.044 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:15.044 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:29:15.045 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:15.045 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:15.045 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:29:15.045 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:15.046 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:15.046 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:15.046 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:29:15.046 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:15.047 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:15.047 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:15.047 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:15.047 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:29:15.048 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:15.048 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:15.048 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:29:15.048 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:15.049 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:15.049 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:15.049 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:29:15.049 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:15.049 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:15.050 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:15.050 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:29:15.050 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:15.050 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:29:15.051 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:29:15.051 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:29:15.051 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:15.051 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:29:15.053 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.825641_203.1 2021-11-11T21:29:20.054 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:20.055 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:20.056 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:29:20.103 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:29:20.104 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:29:20.180 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:29:20.180 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:29:20.181 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:29:20.181 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:29:20.181 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:29:20.181 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:29:20.182 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:20.182 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:20.182 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:29:20.182 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:20.183 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:29:20.183 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:20.183 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:20.183 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:29:20.183 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:20.184 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:29:20.184 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:20.184 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:29:20.184 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:20.185 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:20.185 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:20.185 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:20.185 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:29:20.185 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:20.186 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:20.186 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:29:20.186 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:20.186 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:20.186 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:20.187 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:29:20.187 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:20.187 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:20.187 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:20.188 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:20.188 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:29:20.188 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:20.188 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:20.189 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:29:20.189 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:20.189 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:20.189 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:20.190 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:29:20.190 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:20.190 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:20.190 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:20.191 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:20.191 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:29:20.191 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:20.191 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:20.191 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:29:20.192 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:20.192 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:20.192 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:20.192 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:29:20.193 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:20.193 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:20.193 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:20.193 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:29:20.194 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:20.194 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:29:20.194 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:29:20.194 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:29:20.195 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:20.195 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:29:20.196 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.789745_201.1 2021-11-11T21:29:25.197 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:25.198 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:25.199 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:29:25.263 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:29:25.264 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:29:25.359 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:29:25.360 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:29:25.360 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:29:25.360 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:29:25.361 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:29:25.361 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:29:25.361 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:25.361 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:25.362 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:29:25.362 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:25.362 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:29:25.362 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:25.363 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:25.363 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:29:25.363 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:25.363 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:29:25.363 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:25.364 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:29:25.364 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:25.364 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:25.364 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:25.365 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:25.365 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:29:25.365 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:25.365 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:25.366 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:29:25.366 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:25.366 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:25.366 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:25.367 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:29:25.367 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:25.367 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:25.367 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:25.368 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:25.368 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:29:25.368 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:25.368 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:25.368 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.794229_202.1", 2021-11-11T21:29:25.369 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:25.369 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:25.369 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:25.369 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.794229Z", 2021-11-11T21:29:25.370 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:25.370 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:25.370 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:25.370 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:25.370 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:29:25.371 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:25.371 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:25.371 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:29:25.371 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:25.372 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:25.372 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:25.372 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:29:25.372 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:25.372 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:25.373 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:25.373 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:29:25.373 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:25.373 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:29:25.374 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:29:25.374 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:29:25.374 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:25.374 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:29:25.376 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.794229_202.1 2021-11-11T21:29:30.377 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:30.378 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:30.378 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:29:30.431 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:29:30.431 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:29:30.559 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:29:30.559 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:29:30.560 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:29:30.560 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:29:30.560 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:29:30.560 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:29:30.561 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:30.561 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:30.561 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:29:30.561 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:30.562 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:29:30.562 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:30.562 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:30.562 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:29:30.562 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:30.563 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:29:30.563 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:30.563 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:29:30.563 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:30.564 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:30.564 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:30.564 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:30.564 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:29:30.564 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:30.565 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:30.565 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:29:30.565 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:30.565 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:30.566 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:30.566 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:29:30.567 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:30.567 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:30.568 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:30.568 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:30.568 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:29:30.568 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:30.568 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:30.569 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.582199_241.1", 2021-11-11T21:29:30.569 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:30.569 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:30.569 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:30.569 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.582199Z", 2021-11-11T21:29:30.570 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:30.570 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:30.570 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:30.570 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:30.571 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:29:30.571 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:30.571 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:30.571 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:29:30.572 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:30.572 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:30.572 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:30.572 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:29:30.572 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:30.573 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:30.573 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:30.573 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:29:30.573 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:30.574 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:29:30.574 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:29:30.574 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:29:30.574 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:30.574 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:29:30.575 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665225.582199_241.1 2021-11-11T21:29:35.576 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:35.577 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:35.577 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:29:35.655 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:29:35.655 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:29:36.187 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:29:36.187 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:29:36.188 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:29:36.188 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:29:36.188 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:29:36.188 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:29:36.189 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:36.189 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:36.189 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:29:36.189 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:36.189 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:29:36.190 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:36.190 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:36.190 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:29:36.190 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:36.190 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:29:36.191 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:36.191 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:29:36.191 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:36.191 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:36.192 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:36.192 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:36.192 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:29:36.192 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:36.192 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:36.193 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:29:36.193 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:36.193 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:36.193 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:36.193 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:29:36.194 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:36.194 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:36.194 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:36.194 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:36.195 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:29:36.195 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:36.195 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:36.195 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:29:36.195 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:36.196 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:36.196 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:36.196 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:29:36.196 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:36.197 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:36.197 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:36.197 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:36.197 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:29:36.197 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:36.198 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:36.198 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:29:36.198 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:36.198 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:36.198 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:36.199 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:29:36.199 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:36.199 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:36.199 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:36.200 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:29:36.200 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:36.200 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:29:36.200 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:29:36.200 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:29:36.201 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:36.201 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:29:36.204 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.789745_201.1 2021-11-11T21:29:41.205 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:41.206 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:41.206 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:29:41.255 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:29:41.256 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:29:41.342 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:29:41.342 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:29:41.343 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:29:41.343 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:29:41.343 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:29:41.343 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:29:41.344 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:41.344 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:41.344 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:29:41.345 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:41.345 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:29:41.345 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:41.346 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:41.347 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:29:41.347 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:41.348 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:29:41.348 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:41.348 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:29:41.349 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:41.349 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:41.350 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:41.350 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:41.351 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:29:41.351 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:41.351 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:41.352 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:29:41.352 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:41.353 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:41.353 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:41.354 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:29:41.354 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:41.354 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:41.355 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:41.355 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:41.356 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:29:41.356 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:41.357 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:41.357 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:29:41.358 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:41.358 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:41.359 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:41.359 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:29:41.359 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:41.360 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:41.360 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:41.361 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:41.361 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:29:41.362 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:41.362 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:41.363 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:29:41.363 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:41.364 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:41.364 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:41.365 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:29:41.365 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:41.366 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:41.366 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:41.367 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:29:41.367 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:41.367 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:29:41.368 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:29:41.368 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:29:41.368 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:41.369 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:29:41.370 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:29:46.372 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:46.372 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:29:46.373 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:29:46.423 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:29:46.423 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:29:46.499 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:29:46.500 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:29:46.500 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:29:46.500 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:29:46.501 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:29:46.501 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:29:46.501 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:46.501 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:46.502 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:29:46.502 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:46.502 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:29:46.502 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:46.503 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:46.503 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:29:46.503 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:46.503 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:29:46.504 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:46.504 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:29:46.504 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:46.504 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:46.505 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:46.505 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:46.505 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:29:46.505 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:46.506 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:46.506 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:29:46.506 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:46.506 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:46.506 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:46.507 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:29:46.507 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:46.507 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:46.507 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:46.508 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:46.508 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:29:46.508 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:46.508 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:46.509 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:29:46.509 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:46.510 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:46.510 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:46.510 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:29:46.511 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:46.511 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:46.512 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:46.512 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:46.513 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:29:46.513 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:46.514 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:29:46.514 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:29:46.515 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:29:46.515 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:29:46.515 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:29:46.516 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:29:46.516 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:46.517 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:46.517 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:46.517 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:29:46.518 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:46.518 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:29:46.519 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:29:46.519 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:29:46.519 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:46.519 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:29:46.521 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636665871.647321_298.1 target=1_1636665224.762816_197.1 2021-11-11T21:29:51.523 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_datalog_autotrim ... FAIL 2021-11-11T21:29:51.525 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--sync-from-all=false', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:29:51.525 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'zone', 'modify', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--sync-from-all=false', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:29:51.526 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 zone modify --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --sync-from-all=false --cluster c2 --debug-rgw 0 --debug-ms 0 2021-11-11T21:29:51.598 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:29:51.599 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:29:51.748 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:29:51.747+0000 7f6c4c8bb380 0 failed reading obj info from .rgw.root:zone_info.4a40cc50-357f-4f11-a692-5925070095e8: (2) No such file or directory 2021-11-11T21:29:51.748 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:29:51.747+0000 7f6c4c8bb380 0 WARNING: could not read zone params for zone id=4a40cc50-357f-4f11-a692-5925070095e8 name=test-zone3 2021-11-11T21:29:51.752 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:29:51.752+0000 7f6c4c8bb380 0 failed reading obj info from .rgw.root:zone_info.badb2b8d-7821-4be0-b9ef-60c04d880ece: (2) No such file or directory 2021-11-11T21:29:51.753 INFO:teuthology.orchestra.run.smithi198.stderr:2021-11-11T21:29:51.752+0000 7f6c4c8bb380 0 WARNING: could not read zone params for zone id=badb2b8d-7821-4be0-b9ef-60c04d880ece name=test-zone1 2021-11-11T21:29:51.786 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:29:51.786 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:29:51.787 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone2", 2021-11-11T21:29:51.787 INFO:teuthology.orchestra.run.smithi198.stdout: "domain_root": "test-zone2.rgw.meta:root", 2021-11-11T21:29:51.787 INFO:teuthology.orchestra.run.smithi198.stdout: "control_pool": "test-zone2.rgw.control", 2021-11-11T21:29:51.787 INFO:teuthology.orchestra.run.smithi198.stdout: "gc_pool": "test-zone2.rgw.log:gc", 2021-11-11T21:29:51.788 INFO:teuthology.orchestra.run.smithi198.stdout: "lc_pool": "test-zone2.rgw.log:lc", 2021-11-11T21:29:51.788 INFO:teuthology.orchestra.run.smithi198.stdout: "log_pool": "test-zone2.rgw.log", 2021-11-11T21:29:51.788 INFO:teuthology.orchestra.run.smithi198.stdout: "intent_log_pool": "test-zone2.rgw.log:intent", 2021-11-11T21:29:51.788 INFO:teuthology.orchestra.run.smithi198.stdout: "usage_log_pool": "test-zone2.rgw.log:usage", 2021-11-11T21:29:51.789 INFO:teuthology.orchestra.run.smithi198.stdout: "roles_pool": "test-zone2.rgw.meta:roles", 2021-11-11T21:29:51.789 INFO:teuthology.orchestra.run.smithi198.stdout: "reshard_pool": "test-zone2.rgw.log:reshard", 2021-11-11T21:29:51.789 INFO:teuthology.orchestra.run.smithi198.stdout: "user_keys_pool": "test-zone2.rgw.meta:users.keys", 2021-11-11T21:29:51.789 INFO:teuthology.orchestra.run.smithi198.stdout: "user_email_pool": "test-zone2.rgw.meta:users.email", 2021-11-11T21:29:51.790 INFO:teuthology.orchestra.run.smithi198.stdout: "user_swift_pool": "test-zone2.rgw.meta:users.swift", 2021-11-11T21:29:51.790 INFO:teuthology.orchestra.run.smithi198.stdout: "user_uid_pool": "test-zone2.rgw.meta:users.uid", 2021-11-11T21:29:51.790 INFO:teuthology.orchestra.run.smithi198.stdout: "otp_pool": "test-zone2.rgw.otp", 2021-11-11T21:29:51.790 INFO:teuthology.orchestra.run.smithi198.stdout: "system_key": { 2021-11-11T21:29:51.790 INFO:teuthology.orchestra.run.smithi198.stdout: "access_key": "UBE0V2NN5ZCDGTQQ", 2021-11-11T21:29:51.791 INFO:teuthology.orchestra.run.smithi198.stdout: "secret_key": "H3Z0KUCIZ9a7ljoMNcuIDmouezBjEEOZ" 2021-11-11T21:29:51.791 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:51.791 INFO:teuthology.orchestra.run.smithi198.stdout: "placement_pools": [ 2021-11-11T21:29:51.791 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:51.792 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "default-placement", 2021-11-11T21:29:51.792 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:29:51.792 INFO:teuthology.orchestra.run.smithi198.stdout: "index_pool": "test-zone2.rgw.buckets.index", 2021-11-11T21:29:51.792 INFO:teuthology.orchestra.run.smithi198.stdout: "storage_classes": { 2021-11-11T21:29:51.793 INFO:teuthology.orchestra.run.smithi198.stdout: "STANDARD": { 2021-11-11T21:29:51.793 INFO:teuthology.orchestra.run.smithi198.stdout: "data_pool": "test-zone2.rgw.buckets.data", 2021-11-11T21:29:51.793 INFO:teuthology.orchestra.run.smithi198.stdout: "compression_type": "random" 2021-11-11T21:29:51.793 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:51.794 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:51.794 INFO:teuthology.orchestra.run.smithi198.stdout: "data_extra_pool": "test-zone2.rgw.buckets.non-ec", 2021-11-11T21:29:51.794 INFO:teuthology.orchestra.run.smithi198.stdout: "index_type": 0 2021-11-11T21:29:51.794 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:51.794 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:51.795 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:29:51.795 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:29:51.795 INFO:teuthology.orchestra.run.smithi198.stdout: "notif_pool": "test-zone2.rgw.log:notif" 2021-11-11T21:29:51.795 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:29:51.796 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:29:51.796 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'period', 'update', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:29:51.797 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 period update --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c2 --debug-rgw 0 --debug-ms 0 2021-11-11T21:29:51.866 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:29:51.866 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:29:51.948 INFO:teuthology.orchestra.run.smithi198.stderr:Sending period to new master zone badb2b8d-7821-4be0-b9ef-60c04d880ece 2021-11-11T21:29:52.990 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:29:52.990 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:29:52.991 INFO:teuthology.orchestra.run.smithi198.stdout: "epoch": 5, 2021-11-11T21:29:52.991 INFO:teuthology.orchestra.run.smithi198.stdout: "predecessor_uuid": "ab45263b-38f8-4e1a-9f37-1e83128a94ad", 2021-11-11T21:29:52.991 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": [], 2021-11-11T21:29:52.991 INFO:teuthology.orchestra.run.smithi198.stdout: "period_map": { 2021-11-11T21:29:52.992 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:29:52.992 INFO:teuthology.orchestra.run.smithi198.stdout: "zonegroups": [ 2021-11-11T21:29:52.992 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:52.992 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "test-zonegroup", 2021-11-11T21:29:52.992 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zonegroup", 2021-11-11T21:29:52.993 INFO:teuthology.orchestra.run.smithi198.stdout: "api_name": "", 2021-11-11T21:29:52.993 INFO:teuthology.orchestra.run.smithi198.stdout: "is_master": "true", 2021-11-11T21:29:52.993 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:29:52.993 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:29:52.994 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:29:52.994 INFO:teuthology.orchestra.run.smithi198.stdout: "hostnames": [], 2021-11-11T21:29:52.994 INFO:teuthology.orchestra.run.smithi198.stdout: "hostnames_s3website": [], 2021-11-11T21:29:52.994 INFO:teuthology.orchestra.run.smithi198.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:29:52.995 INFO:teuthology.orchestra.run.smithi198.stdout: "zones": [ 2021-11-11T21:29:52.995 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:52.995 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "31f4b03b-e9bc-4500-a988-e29031fe0663", 2021-11-11T21:29:52.995 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone4", 2021-11-11T21:29:52.995 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:29:52.996 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi198.front.sepia.ceph.com:8001" 2021-11-11T21:29:52.996 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:29:52.996 INFO:teuthology.orchestra.run.smithi198.stdout: "log_meta": "false", 2021-11-11T21:29:52.996 INFO:teuthology.orchestra.run.smithi198.stdout: "log_data": "true", 2021-11-11T21:29:52.997 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:29:52.997 INFO:teuthology.orchestra.run.smithi198.stdout: "read_only": "false", 2021-11-11T21:29:52.997 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_type": "pubsub", 2021-11-11T21:29:52.997 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from_all": "false", 2021-11-11T21:29:52.997 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from": [ 2021-11-11T21:29:52.998 INFO:teuthology.orchestra.run.smithi198.stdout: "test-zone1" 2021-11-11T21:29:52.998 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:29:52.998 INFO:teuthology.orchestra.run.smithi198.stdout: "redirect_zone": "" 2021-11-11T21:29:52.998 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:52.999 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:52.999 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:29:52.999 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone3", 2021-11-11T21:29:52.999 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:29:52.999 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi096.front.sepia.ceph.com:8001" 2021-11-11T21:29:53.000 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:29:53.000 INFO:teuthology.orchestra.run.smithi198.stdout: "log_meta": "false", 2021-11-11T21:29:53.000 INFO:teuthology.orchestra.run.smithi198.stdout: "log_data": "true", 2021-11-11T21:29:53.000 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:29:53.000 INFO:teuthology.orchestra.run.smithi198.stdout: "read_only": "false", 2021-11-11T21:29:53.001 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_type": "", 2021-11-11T21:29:53.001 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from_all": "true", 2021-11-11T21:29:53.001 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from": [], 2021-11-11T21:29:53.001 INFO:teuthology.orchestra.run.smithi198.stdout: "redirect_zone": "" 2021-11-11T21:29:53.002 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:53.002 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:53.002 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:29:53.002 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone2", 2021-11-11T21:29:53.003 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:29:53.003 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi198.front.sepia.ceph.com:8000" 2021-11-11T21:29:53.003 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:29:53.003 INFO:teuthology.orchestra.run.smithi198.stdout: "log_meta": "false", 2021-11-11T21:29:53.003 INFO:teuthology.orchestra.run.smithi198.stdout: "log_data": "true", 2021-11-11T21:29:53.004 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:29:53.004 INFO:teuthology.orchestra.run.smithi198.stdout: "read_only": "false", 2021-11-11T21:29:53.004 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_type": "", 2021-11-11T21:29:53.004 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from_all": "false", 2021-11-11T21:29:53.005 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from": [], 2021-11-11T21:29:53.005 INFO:teuthology.orchestra.run.smithi198.stdout: "redirect_zone": "" 2021-11-11T21:29:53.005 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:53.005 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:53.005 INFO:teuthology.orchestra.run.smithi198.stdout: "id": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:29:53.006 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "test-zone1", 2021-11-11T21:29:53.006 INFO:teuthology.orchestra.run.smithi198.stdout: "endpoints": [ 2021-11-11T21:29:53.006 INFO:teuthology.orchestra.run.smithi198.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:29:53.006 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:29:53.007 INFO:teuthology.orchestra.run.smithi198.stdout: "log_meta": "false", 2021-11-11T21:29:53.007 INFO:teuthology.orchestra.run.smithi198.stdout: "log_data": "true", 2021-11-11T21:29:53.007 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:29:53.007 INFO:teuthology.orchestra.run.smithi198.stdout: "read_only": "false", 2021-11-11T21:29:53.007 INFO:teuthology.orchestra.run.smithi198.stdout: "tier_type": "", 2021-11-11T21:29:53.008 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from_all": "true", 2021-11-11T21:29:53.008 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_from": [], 2021-11-11T21:29:53.008 INFO:teuthology.orchestra.run.smithi198.stdout: "redirect_zone": "" 2021-11-11T21:29:53.008 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:53.008 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:29:53.009 INFO:teuthology.orchestra.run.smithi198.stdout: "placement_targets": [ 2021-11-11T21:29:53.009 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:53.009 INFO:teuthology.orchestra.run.smithi198.stdout: "name": "default-placement", 2021-11-11T21:29:53.009 INFO:teuthology.orchestra.run.smithi198.stdout: "tags": [], 2021-11-11T21:29:53.010 INFO:teuthology.orchestra.run.smithi198.stdout: "storage_classes": [ 2021-11-11T21:29:53.010 INFO:teuthology.orchestra.run.smithi198.stdout: "STANDARD" 2021-11-11T21:29:53.010 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:29:53.010 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:53.011 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:29:53.011 INFO:teuthology.orchestra.run.smithi198.stdout: "default_placement": "default-placement", 2021-11-11T21:29:53.011 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:29:53.011 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_policy": { 2021-11-11T21:29:53.011 INFO:teuthology.orchestra.run.smithi198.stdout: "groups": [] 2021-11-11T21:29:53.012 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:53.012 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:53.012 INFO:teuthology.orchestra.run.smithi198.stdout: ], 2021-11-11T21:29:53.012 INFO:teuthology.orchestra.run.smithi198.stdout: "short_zone_ids": [ 2021-11-11T21:29:53.012 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:53.014 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "31f4b03b-e9bc-4500-a988-e29031fe0663", 2021-11-11T21:29:53.014 INFO:teuthology.orchestra.run.smithi198.stdout: "val": 2936728019 2021-11-11T21:29:53.014 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:53.014 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:53.015 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:29:53.015 INFO:teuthology.orchestra.run.smithi198.stdout: "val": 986920242 2021-11-11T21:29:53.015 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:53.015 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:53.016 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:29:53.016 INFO:teuthology.orchestra.run.smithi198.stdout: "val": 1380659451 2021-11-11T21:29:53.016 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:53.016 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:29:53.017 INFO:teuthology.orchestra.run.smithi198.stdout: "key": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:29:53.017 INFO:teuthology.orchestra.run.smithi198.stdout: "val": 1844113099 2021-11-11T21:29:53.017 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:53.017 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:29:53.018 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:53.018 INFO:teuthology.orchestra.run.smithi198.stdout: "master_zonegroup": "test-zonegroup", 2021-11-11T21:29:53.018 INFO:teuthology.orchestra.run.smithi198.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:29:53.018 INFO:teuthology.orchestra.run.smithi198.stdout: "period_config": { 2021-11-11T21:29:53.018 INFO:teuthology.orchestra.run.smithi198.stdout: "bucket_quota": { 2021-11-11T21:29:53.019 INFO:teuthology.orchestra.run.smithi198.stdout: "enabled": false, 2021-11-11T21:29:53.019 INFO:teuthology.orchestra.run.smithi198.stdout: "check_on_raw": false, 2021-11-11T21:29:53.019 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size": -1, 2021-11-11T21:29:53.019 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size_kb": 0, 2021-11-11T21:29:53.020 INFO:teuthology.orchestra.run.smithi198.stdout: "max_objects": -1 2021-11-11T21:29:53.020 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:53.020 INFO:teuthology.orchestra.run.smithi198.stdout: "user_quota": { 2021-11-11T21:29:53.020 INFO:teuthology.orchestra.run.smithi198.stdout: "enabled": false, 2021-11-11T21:29:53.021 INFO:teuthology.orchestra.run.smithi198.stdout: "check_on_raw": false, 2021-11-11T21:29:53.021 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size": -1, 2021-11-11T21:29:53.021 INFO:teuthology.orchestra.run.smithi198.stdout: "max_size_kb": 0, 2021-11-11T21:29:53.021 INFO:teuthology.orchestra.run.smithi198.stdout: "max_objects": -1 2021-11-11T21:29:53.021 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:29:53.022 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:29:53.022 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:29:53.022 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_name": "test-realm", 2021-11-11T21:29:53.022 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:29:53.023 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:29:53.023 INFO:rgw_multi.tests:Set sync_from_all flag on zone test-zone2 to false 2021-11-11T21:30:53.025 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-45 2021-11-11T21:30:53.099 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:30:53.100 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:30:53.101 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:30:53.320 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:30:53.321 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:30:53.380 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:30:53.403 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:30:53.404 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:30:53.404 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:30:53.404 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:30:53.405 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:30:53.405 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:30:53.405 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:30:53.405 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:30:53.406 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:30:53.406 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:30:53.406 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636666253.051934_329.1", 2021-11-11T21:30:53.406 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:30:53.051934Z" 2021-11-11T21:30:53.407 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:30:53.407 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:30:53.407 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:30:53.407 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:30:53.408 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:30:53.408 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:30:53.409 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:30:53.409 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:30:53.410 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:30:53.411 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:30:53.459 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:30:53.459 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:30:53.544 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:30:53.544 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:30:53.544 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:30:53.545 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:30:53.545 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:30:53.545 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:30:53.545 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:30:53.546 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:30:53.546 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:30:53.546 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:30:53.546 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:30:53.547 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:30:53.547 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:30:53.547 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:30:53.547 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:30:53.548 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:30:53.548 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:30:53.548 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:30:53.549 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:30:53.549 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:53.549 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:30:53.549 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:30:53.550 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:30:53.550 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:30:53.550 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:30:53.550 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:30:53.551 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:30:53.551 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:30:53.551 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:30:53.551 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:30:53.551 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:30:53.552 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:53.552 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:30:53.552 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:30:53.552 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:30:53.552 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:30:53.553 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:30:53.553 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:30:53.553 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:30:53.553 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:30:53.554 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:30:53.554 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:30:53.554 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:30:53.554 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:53.554 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:30:53.555 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:30:53.555 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:30:53.555 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:30:53.555 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:30:53.556 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:30:53.556 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:30:53.556 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:30:53.556 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:30:53.556 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:30:53.557 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:30:53.557 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:53.557 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:53.557 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:30:53.558 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:30:53.558 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:30:53.558 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:30:53.558 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:30:53.558 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:53.559 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:30:53.560 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:30:58.562 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:30:58.563 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:30:58.563 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:30:58.613 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:30:58.613 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:30:58.694 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:30:58.695 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:30:58.695 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:30:58.695 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:30:58.695 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:30:58.696 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:30:58.696 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:30:58.696 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:30:58.696 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:30:58.697 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:30:58.697 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:30:58.697 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:30:58.697 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:30:58.697 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:30:58.698 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:30:58.698 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:30:58.698 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:30:58.698 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:30:58.699 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:30:58.699 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:58.699 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:30:58.699 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:30:58.699 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:30:58.700 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:30:58.700 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:30:58.700 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:30:58.700 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:30:58.701 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:30:58.701 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:30:58.701 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:30:58.702 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:30:58.702 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:58.702 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:30:58.702 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:30:58.703 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:30:58.703 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:30:58.703 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:30:58.703 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:30:58.703 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:30:58.704 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:30:58.705 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:30:58.705 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:30:58.705 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:30:58.706 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:58.706 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:30:58.706 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:30:58.706 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:30:58.706 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:30:58.707 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:30:58.707 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:30:58.707 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:30:58.707 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:30:58.708 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:30:58.708 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:30:58.708 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:30:58.708 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:58.708 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:58.709 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:30:58.709 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:30:58.709 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:30:58.709 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:30:58.710 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:30:58.710 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:30:58.710 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:30:58.711 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:31:03.713 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:03.714 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:03.715 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:31:03.764 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:31:03.764 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:31:03.839 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:31:03.839 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:31:03.840 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:31:03.840 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:31:03.840 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:31:03.840 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:31:03.841 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:03.841 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:03.841 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:31:03.841 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:03.841 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:31:03.842 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:03.842 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:03.842 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:31:03.842 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:03.843 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:31:03.843 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:03.843 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:31:03.843 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:03.844 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:03.844 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:03.844 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:03.844 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:31:03.845 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:03.845 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:03.845 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:31:03.845 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:03.846 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:03.846 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:03.846 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:31:03.846 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:03.847 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:03.847 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:03.847 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:03.848 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:31:03.848 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:03.848 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:03.849 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:31:03.849 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:03.849 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:03.850 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:03.850 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:31:03.850 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:03.851 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:03.851 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:03.851 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:03.851 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:31:03.852 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:03.852 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:03.852 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:31:03.852 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:03.853 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:03.853 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:03.853 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:31:03.853 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:03.854 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:03.854 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:03.854 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:31:03.854 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:03.855 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:31:03.855 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:31:03.855 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:31:03.856 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:03.856 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:31:03.857 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:31:08.859 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:08.859 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:08.860 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:31:08.909 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:31:08.910 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:31:08.993 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:31:08.993 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:31:08.993 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:31:08.994 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:31:08.994 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:31:08.994 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:31:08.994 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:08.995 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:08.995 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:31:08.995 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:08.995 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:31:08.995 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:08.996 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:08.996 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:31:08.996 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:08.996 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:31:08.996 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:08.997 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:31:08.997 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:08.997 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:08.997 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:08.998 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:08.998 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:31:08.998 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:08.998 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:08.998 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:31:08.999 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:08.999 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:08.999 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:08.999 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:31:09.000 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:09.000 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:09.000 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:09.000 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:09.000 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:31:09.001 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:09.001 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:09.001 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:31:09.001 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:09.002 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:09.002 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:09.002 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:31:09.002 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:09.002 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:09.003 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:09.003 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:09.003 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:31:09.003 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:09.004 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:09.004 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:31:09.004 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:09.004 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:09.004 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:09.005 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:31:09.005 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:09.005 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:09.005 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:09.005 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:31:09.006 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:09.006 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:31:09.006 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:31:09.006 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:31:09.007 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:09.007 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:31:09.008 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:31:14.009 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:14.010 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:14.011 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:31:14.059 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:31:14.059 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:31:14.137 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:31:14.137 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:31:14.137 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:31:14.138 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:31:14.138 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:31:14.138 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:31:14.138 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:14.139 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:14.139 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:31:14.139 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:14.139 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:31:14.140 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:14.140 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:14.140 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:31:14.140 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:14.140 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:31:14.141 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:14.141 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:31:14.141 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:14.141 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:14.142 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:14.142 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:14.142 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:31:14.142 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:14.142 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:14.143 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:31:14.143 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:14.143 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:14.143 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:14.144 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:31:14.144 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:14.144 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:14.144 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:14.144 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:14.145 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:31:14.145 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:14.145 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:14.145 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:31:14.146 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:14.146 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:14.146 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:14.146 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:31:14.146 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:14.147 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:14.147 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:14.147 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:14.147 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:31:14.148 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:14.148 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:14.148 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:31:14.148 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:14.148 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:14.149 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:14.149 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:31:14.149 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:14.149 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:14.149 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:14.150 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:31:14.150 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:14.150 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:31:14.150 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:31:14.150 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:31:14.151 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:14.151 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:31:14.152 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:31:19.153 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:19.153 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:19.154 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:31:19.203 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:31:19.203 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:31:19.286 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:31:19.286 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:31:19.286 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:31:19.287 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:31:19.287 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:31:19.287 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:31:19.287 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:19.287 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:19.288 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:31:19.288 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:19.288 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:31:19.288 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:19.289 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:19.289 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:31:19.289 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:19.289 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:31:19.290 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:19.290 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:31:19.290 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:19.290 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:19.290 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:19.291 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:19.291 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:31:19.291 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:19.291 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:19.292 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:31:19.292 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:19.292 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:19.292 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:19.292 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:31:19.293 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:19.293 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:19.293 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:19.293 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:19.294 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:31:19.294 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:19.294 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:19.294 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:31:19.294 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:19.295 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:19.295 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:19.295 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:31:19.295 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:19.295 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:19.296 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:19.296 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:19.296 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:31:19.296 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:19.297 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:19.297 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:31:19.297 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:19.297 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:19.298 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:19.298 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:31:19.298 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:19.298 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:19.299 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:19.299 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:31:19.299 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:19.299 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:31:19.299 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:31:19.300 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:31:19.300 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:19.300 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:31:19.301 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:31:24.302 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:24.303 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:24.304 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:31:24.352 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:31:24.352 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:31:24.438 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:31:24.438 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:31:24.438 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:31:24.439 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:31:24.439 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:31:24.439 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:31:24.439 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:24.439 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:24.440 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:31:24.440 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:24.440 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:31:24.440 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:24.441 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:24.441 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:31:24.441 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:24.441 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:31:24.441 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:24.442 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:31:24.442 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:24.442 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:24.442 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:24.443 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:24.443 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:31:24.443 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:24.443 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:24.444 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:31:24.444 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:24.444 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:24.444 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:24.444 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:31:24.445 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:24.445 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:24.445 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:24.445 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:24.445 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:31:24.446 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:24.446 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:24.446 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:31:24.446 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:24.447 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:24.447 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:24.447 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:31:24.447 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:24.447 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:24.448 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:24.448 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:24.448 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:31:24.448 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:24.448 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:24.449 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:31:24.449 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:24.449 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:24.449 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:24.450 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:31:24.450 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:24.450 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:24.450 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:24.450 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:31:24.451 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:24.451 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:31:24.451 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:31:24.451 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:31:24.452 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:24.452 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:31:24.453 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:31:29.454 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:29.455 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:29.455 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:31:29.505 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:31:29.505 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:31:29.579 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:31:29.579 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:31:29.579 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:31:29.580 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:31:29.580 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:31:29.580 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:31:29.580 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:29.581 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:29.581 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:31:29.581 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:29.581 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:31:29.582 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:29.582 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:29.582 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:31:29.582 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:29.582 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:31:29.583 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:29.583 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:31:29.583 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:29.583 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:29.584 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:29.584 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:29.584 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:31:29.584 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:29.585 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:29.585 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:31:29.585 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:29.585 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:29.585 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:29.586 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:31:29.586 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:29.586 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:29.586 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:29.587 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:29.587 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:31:29.587 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:29.587 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:29.587 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:31:29.588 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:29.588 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:29.588 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:29.588 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:31:29.589 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:29.589 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:29.589 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:29.589 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:29.589 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:31:29.590 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:29.590 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:29.590 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:31:29.590 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:29.591 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:29.591 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:29.591 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:31:29.591 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:29.591 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:29.592 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:29.592 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:31:29.592 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:29.592 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:31:29.593 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:31:29.593 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:31:29.593 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:29.593 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:31:29.595 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:31:34.596 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:34.597 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:34.598 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:31:34.645 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:31:34.646 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:31:34.721 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:31:34.722 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:31:34.722 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:31:34.722 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:31:34.723 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:31:34.723 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:31:34.723 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:34.723 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:34.723 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:31:34.724 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:34.724 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:31:34.724 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:34.724 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:34.725 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:31:34.725 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:34.725 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:31:34.725 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:34.725 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:31:34.726 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:34.726 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:34.726 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:34.726 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:34.727 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:31:34.727 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:34.727 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:34.727 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:31:34.727 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:34.728 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:34.728 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:34.728 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:31:34.728 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:34.728 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:34.729 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:34.729 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:34.729 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:31:34.729 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:34.729 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:34.730 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:31:34.730 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:34.730 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:34.730 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:34.730 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:31:34.731 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:34.731 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:34.731 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:34.731 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:34.731 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:31:34.732 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:34.732 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:34.732 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:31:34.732 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:34.732 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:34.733 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:34.733 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:31:34.733 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:34.733 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:34.734 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:34.734 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:31:34.734 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:34.734 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:31:34.734 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:31:34.735 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:31:34.735 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:34.735 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:31:34.736 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:31:39.737 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:39.738 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:39.739 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:31:39.788 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:31:39.788 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:31:39.867 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:31:39.867 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:31:39.867 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:31:39.867 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:31:39.868 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:31:39.868 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:31:39.868 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:39.868 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:39.869 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:31:39.869 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:39.869 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:31:39.869 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:39.870 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:39.870 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:31:39.870 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:39.870 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:31:39.871 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:39.871 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:31:39.871 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:39.871 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:39.871 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:39.872 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:39.872 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:31:39.872 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:39.872 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:39.873 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:31:39.873 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:39.873 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:39.873 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:39.874 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:31:39.874 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:39.874 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:39.874 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:39.874 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:39.875 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:31:39.875 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:39.875 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:39.875 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:31:39.876 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:39.876 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:39.876 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:39.876 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:31:39.877 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:39.877 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:39.877 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:39.877 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:39.877 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:31:39.878 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:39.878 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:39.878 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:31:39.878 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:39.879 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:39.879 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:39.879 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:31:39.879 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:39.880 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:39.880 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:39.880 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:31:39.880 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:39.881 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:31:39.881 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:31:39.881 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:31:39.881 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:39.881 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:31:39.883 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:31:44.884 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:44.885 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:44.886 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:31:44.937 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:31:44.937 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:31:45.008 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:31:45.009 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:31:45.009 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:31:45.009 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:31:45.010 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:31:45.010 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:31:45.011 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:45.011 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:45.012 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:31:45.012 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:45.013 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:31:45.013 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:45.013 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:45.014 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:31:45.014 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:45.015 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:31:45.015 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:45.016 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:31:45.016 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:45.016 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:45.017 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:45.017 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:45.018 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:31:45.018 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:45.018 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:45.019 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:31:45.019 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:45.020 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:45.020 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:45.020 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:31:45.021 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:45.021 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:45.022 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:45.022 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:45.023 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:31:45.023 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:45.023 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:45.024 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:31:45.024 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:45.024 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:45.025 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:45.025 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:31:45.025 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:45.026 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:45.026 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:45.027 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:45.027 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:31:45.027 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:45.028 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:45.028 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:31:45.028 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:45.029 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:45.029 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:45.030 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:31:45.030 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:45.031 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:45.031 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:45.031 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:31:45.032 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:45.032 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:31:45.033 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:31:45.033 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:31:45.033 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:45.034 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:31:45.036 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:31:50.037 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:50.038 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:50.039 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:31:50.087 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:31:50.088 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:31:50.160 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:31:50.160 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:31:50.160 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:31:50.161 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:31:50.161 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:31:50.161 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:31:50.161 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:50.161 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:50.162 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:31:50.162 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:50.162 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:31:50.163 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:50.163 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:50.163 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:31:50.163 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:50.164 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:31:50.164 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:50.164 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:31:50.164 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:50.165 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:50.165 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:50.165 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:50.165 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:31:50.165 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:50.166 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:50.166 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:31:50.166 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:50.166 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:50.167 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:50.167 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:31:50.167 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:50.167 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:50.167 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:50.168 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:50.168 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:31:50.168 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:50.168 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:50.169 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.762816_197.1", 2021-11-11T21:31:50.169 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:50.169 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:50.169 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:50.169 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.762816Z", 2021-11-11T21:31:50.170 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:50.170 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:50.170 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:50.170 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:50.171 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:31:50.171 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:50.171 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:50.171 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:31:50.171 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:50.172 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:50.172 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:50.172 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:31:50.172 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:50.173 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:50.173 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:50.173 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:31:50.173 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:50.173 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:31:50.174 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:31:50.174 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:31:50.174 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:50.175 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:31:50.176 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.762816_197.1 2021-11-11T21:31:55.178 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:55.179 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:31:55.180 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:31:55.222 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:31:55.223 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:31:55.289 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:31:55.290 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:31:55.290 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:31:55.291 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:31:55.291 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:31:55.291 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:31:55.291 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:55.292 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:55.292 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:31:55.292 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:55.292 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:31:55.293 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:55.293 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:55.294 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:31:55.294 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:55.294 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:31:55.294 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:55.295 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:31:55.295 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:55.295 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:55.295 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:55.295 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:55.296 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:31:55.296 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:55.297 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:55.297 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:31:55.297 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:55.297 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:55.298 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:55.298 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:31:55.298 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:55.299 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:55.299 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:55.299 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:55.299 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:31:55.300 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:55.300 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:55.300 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:31:55.300 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:55.301 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:55.301 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:55.301 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:31:55.301 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:55.302 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:55.302 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:55.302 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:31:55.302 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:31:55.303 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:31:55.303 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:31:55.303 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:31:55.304 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:31:55.304 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:31:55.304 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:31:55.304 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:31:55.305 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:31:55.305 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:55.305 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:55.305 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:31:55.306 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:31:55.306 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:31:55.306 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:31:55.306 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:31:55.307 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:31:55.307 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:31:55.308 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.789745_201.1 2021-11-11T21:32:00.310 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:00.310 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:00.311 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:00.354 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:00.354 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:00.420 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:00.421 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:00.421 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:00.421 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:00.422 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:00.422 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:00.422 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:00.422 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:00.423 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:00.423 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:00.423 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:00.423 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:00.423 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:00.424 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:00.424 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:00.424 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:00.424 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:00.425 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:00.425 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:00.425 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:00.426 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:00.426 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:00.426 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:00.426 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:00.426 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:00.427 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:00.427 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:00.427 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:00.427 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:00.428 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:00.428 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:00.428 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:00.428 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:00.429 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:00.429 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:00.429 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:00.429 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:00.430 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:32:00.430 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:00.430 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:00.430 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:00.430 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:32:00.431 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:00.431 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:00.431 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:00.432 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:00.432 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:00.432 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:00.432 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:00.432 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:00.433 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:00.433 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:00.433 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:00.433 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:00.434 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:00.434 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:00.434 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:00.434 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:00.435 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:00.435 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:00.435 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:00.435 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:00.435 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:00.436 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:00.437 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:32:05.439 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:05.440 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:05.441 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:05.485 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:05.486 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:05.575 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:05.576 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:05.576 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:05.576 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:05.576 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:05.577 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:05.577 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:05.577 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:05.577 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:05.578 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:05.578 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:05.578 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:05.579 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:05.579 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:05.579 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:05.580 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:05.580 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:05.580 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:05.580 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:05.580 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:05.581 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:05.581 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:05.581 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:05.581 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:05.582 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:05.582 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:05.583 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:05.583 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:05.584 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:05.584 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:05.584 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:05.585 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:05.585 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:05.586 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:05.586 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:05.587 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:05.587 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:05.588 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:32:05.588 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:05.589 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:05.589 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:05.589 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:32:05.590 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:05.590 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:05.591 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:05.591 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:05.592 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:05.592 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:05.592 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:05.593 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:05.593 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:05.594 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:05.594 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:05.595 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:05.595 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:05.596 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:05.596 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:05.597 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:05.597 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:05.597 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:05.598 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:05.598 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:05.599 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:05.599 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:05.601 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.830110_204.1 2021-11-11T21:32:10.603 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:10.604 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:10.604 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:10.708 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:10.708 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:11.205 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:11.205 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:11.206 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:11.206 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:11.206 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:11.206 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:11.207 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:11.207 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:11.207 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:11.207 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:11.208 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:11.208 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:11.208 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:11.208 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:11.209 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:11.209 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:11.209 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:11.209 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:11.209 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:11.210 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:11.210 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:11.210 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:11.210 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:11.211 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:11.211 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:11.211 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:11.211 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:11.212 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:11.212 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:11.212 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:11.212 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:11.213 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:11.213 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:11.213 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:11.213 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:11.213 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:11.214 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:11.214 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.114178_225.1", 2021-11-11T21:32:11.214 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:11.214 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:11.215 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:11.215 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.114178Z", 2021-11-11T21:32:11.215 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:11.215 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:11.216 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:11.216 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:11.216 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:11.216 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:11.216 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:11.217 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:11.217 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:11.217 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:11.218 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:11.218 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:11.218 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:11.219 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:11.219 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:11.219 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:11.219 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:11.220 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:11.220 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:11.220 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:11.220 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:11.220 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:11.222 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.114178_225.1 2021-11-11T21:32:16.223 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:16.224 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:16.225 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:16.268 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:16.268 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:16.432 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:16.432 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:16.432 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:16.432 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:16.433 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:16.433 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:16.433 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:16.433 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:16.434 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:16.434 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:16.434 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:16.435 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:16.435 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:16.435 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:16.435 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:16.436 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:16.436 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:16.436 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:16.436 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:16.437 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:16.437 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:16.437 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:16.437 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:16.437 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:16.438 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:16.438 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:16.438 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:16.439 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:16.439 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:16.439 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:16.439 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:16.440 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:16.440 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:16.440 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:16.440 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:16.440 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:16.441 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:16.441 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:32:16.441 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:16.441 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:16.441 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:16.442 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:32:16.442 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:16.442 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:16.442 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:16.443 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:16.443 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:16.443 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:16.443 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:16.443 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:16.444 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:16.444 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:16.444 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:16.444 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:16.445 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:16.445 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:16.445 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:16.445 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:16.445 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:16.446 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:16.446 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:16.446 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:16.446 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:16.447 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:16.448 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.783136_199.1 2021-11-11T21:32:21.449 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:21.450 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:21.451 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:21.495 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:21.495 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:21.575 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:21.575 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:21.575 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:21.575 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:21.576 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:21.576 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:21.576 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:21.576 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:21.577 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:21.577 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:21.577 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:21.577 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:21.577 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:21.578 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:21.578 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:21.578 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:21.578 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:21.578 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:21.579 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:21.579 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:21.579 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:21.579 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:21.580 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:21.580 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:21.580 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:21.581 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:21.581 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:21.581 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:21.581 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:21.581 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:21.582 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:21.582 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:21.582 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:21.582 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:21.583 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:21.583 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:21.583 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:21.583 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.838825_206.1", 2021-11-11T21:32:21.583 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:21.584 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:21.584 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:21.584 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.838825Z", 2021-11-11T21:32:21.584 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:21.585 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:21.585 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:21.585 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:21.585 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:21.586 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:21.586 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:21.586 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:21.586 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:21.587 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:21.587 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:21.587 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:21.587 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:21.588 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:21.588 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:21.588 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:21.588 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:21.589 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:21.589 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:21.590 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:21.590 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:21.590 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:21.592 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.838825_206.1 2021-11-11T21:32:26.593 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:26.594 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:26.595 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:26.679 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:26.679 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:26.778 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:26.778 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:26.778 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:26.779 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:26.779 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:26.779 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:26.779 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:26.780 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:26.780 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:26.780 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:26.780 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:26.780 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:26.781 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:26.781 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:26.781 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:26.781 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:26.782 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:26.782 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:26.782 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:26.782 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:26.783 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:26.783 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:26.783 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:26.783 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:26.783 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:26.784 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:26.784 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:26.784 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:26.784 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:26.785 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:26.785 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:26.785 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:26.785 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:26.785 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:26.786 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:26.786 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:26.786 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:26.786 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:32:26.787 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:26.787 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:26.787 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:26.787 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:32:26.787 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:26.788 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:26.788 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:26.788 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:26.788 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:26.789 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:26.789 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:26.789 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:26.789 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:26.789 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:26.790 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:26.790 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:26.790 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:26.790 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:26.791 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:26.791 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:26.791 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:26.791 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:26.792 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:26.792 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:26.792 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:26.792 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:26.793 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.783136_199.1 2021-11-11T21:32:31.794 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:31.795 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:31.796 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:31.864 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:31.864 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:32.314 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:32.314 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:32.314 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:32.315 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:32.315 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:32.315 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:32.315 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:32.315 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:32.316 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:32.316 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:32.316 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:32.316 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:32.317 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:32.317 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:32.317 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:32.317 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:32.317 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:32.318 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:32.318 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:32.318 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:32.318 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:32.319 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:32.319 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:32.319 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:32.319 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:32.320 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:32.320 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:32.320 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:32.320 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:32.321 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:32.321 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:32.321 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:32.321 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:32.321 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:32.322 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:32.322 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:32.322 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:32.322 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:32:32.323 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:32.323 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:32.323 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:32.323 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:32:32.324 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:32.324 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:32.324 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:32.324 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:32.324 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:32.325 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:32.325 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:32.325 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:32.325 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:32.325 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:32.326 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:32.326 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:32.326 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:32.326 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:32.327 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:32.327 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:32.327 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:32.327 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:32.327 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:32.328 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:32.328 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:32.328 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:32.329 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:32:37.330 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:37.331 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:37.332 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:37.420 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:37.421 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:37.591 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:37.592 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:37.592 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:37.592 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:37.592 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:37.593 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:37.593 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:37.593 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:37.593 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:37.594 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:37.594 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:37.594 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:37.594 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:37.595 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:37.595 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:37.595 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:37.595 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:37.596 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:37.596 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:37.596 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:37.596 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:37.597 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:37.597 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:37.597 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:37.597 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:37.598 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:37.598 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:37.598 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:37.598 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:37.599 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:37.599 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:37.599 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:37.599 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:37.600 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:37.600 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:37.600 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:37.600 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:37.601 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.914354_210.1", 2021-11-11T21:32:37.601 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:37.601 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:37.601 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:37.602 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.914354Z", 2021-11-11T21:32:37.602 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:37.602 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:37.602 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:37.603 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:37.603 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:37.603 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:37.603 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:37.604 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:37.604 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:37.604 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:37.604 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:37.605 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:37.605 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:37.605 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:37.605 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:37.606 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:37.606 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:37.606 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:37.606 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:37.607 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:37.607 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:37.607 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:37.609 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.914354_210.1 2021-11-11T21:32:42.610 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:42.611 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:42.612 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:42.701 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:42.701 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:43.232 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:43.232 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:43.232 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:43.233 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:43.233 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:43.233 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:43.233 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:43.234 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:43.234 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:43.234 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:43.234 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:43.235 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:43.235 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:43.235 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:43.235 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:43.236 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:43.236 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:43.236 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:43.236 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:43.237 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:43.237 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:43.237 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:43.237 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:43.238 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:43.238 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:43.238 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:43.238 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:43.239 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:43.239 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:43.239 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:43.239 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:43.239 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:43.240 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:43.240 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:43.240 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:43.240 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:43.241 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:43.241 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:32:43.241 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:43.241 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:43.242 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:43.242 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:32:43.242 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:43.242 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:43.242 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:43.243 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:43.243 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:43.243 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:43.243 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:43.244 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:43.244 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:43.244 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:43.244 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:43.244 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:43.245 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:43.245 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:43.245 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:43.245 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:43.246 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:43.246 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:43.246 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:43.246 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:43.246 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:43.247 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:43.247 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.831934_205.1 2021-11-11T21:32:48.248 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:48.249 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:48.250 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:48.307 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:48.308 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:48.825 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:48.825 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:48.826 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:48.826 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:48.826 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:48.826 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:48.827 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:48.827 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:48.827 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:48.827 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:48.828 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:48.828 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:48.828 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:48.828 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:48.828 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:48.829 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:48.829 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:48.829 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:48.829 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:48.830 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:48.830 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:48.830 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:48.830 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:48.831 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:48.831 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:48.831 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:48.831 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:48.831 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:48.832 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:48.832 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:48.832 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:48.832 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:48.833 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:48.833 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:48.833 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:48.833 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:48.834 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:48.834 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:32:48.834 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:48.834 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:48.834 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:48.835 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:32:48.835 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:48.835 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:48.835 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:48.836 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:48.836 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:48.836 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:48.836 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:48.836 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:48.837 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:48.837 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:48.837 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:48.837 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:48.838 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:48.838 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:48.838 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:48.838 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:48.838 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:48.839 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:48.839 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:48.840 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:48.840 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:48.840 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:48.842 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.825641_203.1 2021-11-11T21:32:53.843 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:53.844 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:53.845 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:53.893 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:53.894 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:54.235 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:54.235 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:54.235 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:54.236 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:54.236 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:54.236 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:54.236 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:54.237 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:54.237 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:54.237 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:54.238 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:54.238 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:54.238 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:54.238 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:54.238 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:54.239 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:54.239 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:54.239 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:54.240 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:54.240 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:54.240 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:54.240 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:54.240 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:54.241 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:54.241 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:54.241 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:54.241 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:54.242 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:54.242 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:54.242 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:54.242 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:54.243 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:54.243 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:54.243 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:54.243 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:54.244 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:54.244 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:54.244 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:32:54.244 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:54.244 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:54.245 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:54.245 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:32:54.245 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:54.245 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:54.246 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:54.246 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:54.246 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:54.246 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:54.246 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:54.247 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:54.247 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:54.247 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:54.247 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:54.248 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:54.248 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:54.248 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:54.248 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:54.248 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:54.249 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:54.249 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:54.249 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:54.249 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:54.250 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:54.250 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:54.250 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.787834_200.1 2021-11-11T21:32:59.252 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:59.253 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:32:59.253 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:32:59.299 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:32:59.299 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:32:59.388 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:32:59.388 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:32:59.388 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:32:59.389 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:32:59.389 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:32:59.389 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:32:59.389 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:59.390 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:59.390 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:32:59.390 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:59.390 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:32:59.391 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:59.391 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:59.391 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:32:59.391 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:59.391 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:32:59.392 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:59.392 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:32:59.392 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:59.392 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:59.393 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:59.393 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:59.393 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:32:59.393 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:59.394 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:59.394 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:32:59.394 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:59.394 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:59.395 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:59.395 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:32:59.395 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:59.395 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:59.396 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:59.396 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:59.396 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:32:59.396 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:59.396 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:59.397 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:32:59.397 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:59.397 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:59.397 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:59.397 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:32:59.398 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:59.398 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:59.398 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:59.398 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:32:59.399 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:32:59.399 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:32:59.399 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:32:59.399 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:32:59.400 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:32:59.400 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:32:59.400 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:32:59.400 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:32:59.400 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:32:59.401 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:59.401 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:59.401 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:32:59.401 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:32:59.402 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:32:59.402 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:32:59.402 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:32:59.402 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:32:59.403 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:32:59.404 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.783136_199.1 2021-11-11T21:33:04.406 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:04.406 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:04.407 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:33:04.452 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:33:04.453 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:33:04.529 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:33:04.529 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:33:04.530 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:33:04.530 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:33:04.530 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:33:04.530 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:33:04.531 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:04.531 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:04.531 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:33:04.531 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:04.532 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:33:04.532 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:04.532 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:04.532 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:33:04.533 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:04.533 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:33:04.533 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:04.533 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:33:04.533 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:04.534 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:04.534 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:04.534 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:04.535 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:33:04.535 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:04.535 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:04.535 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:33:04.536 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:04.536 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:04.536 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:04.536 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:33:04.537 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:04.537 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:04.537 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:04.537 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:04.537 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:33:04.538 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:04.538 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:04.538 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:33:04.538 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:04.539 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:04.539 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:04.539 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:33:04.539 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:04.539 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:04.540 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:04.540 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:04.540 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:33:04.540 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:04.541 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:04.541 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:33:04.541 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:04.541 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:04.541 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:04.542 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:33:04.542 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:04.542 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:04.543 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:04.543 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:33:04.543 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:04.543 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:33:04.543 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:33:04.544 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:33:04.544 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:04.544 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:33:04.545 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.787834_200.1 2021-11-11T21:33:09.547 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:09.548 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:09.549 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:33:09.633 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:33:09.634 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:33:09.784 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:33:09.785 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:33:09.785 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:33:09.785 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:33:09.786 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:33:09.786 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:33:09.786 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:09.787 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:09.787 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:33:09.787 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:09.787 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:33:09.787 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:09.788 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:09.788 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:33:09.788 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:09.789 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:33:09.790 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:09.790 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:33:09.790 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:09.791 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:09.792 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:09.792 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:09.792 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:33:09.792 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:09.793 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:09.794 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:33:09.795 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:09.795 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:09.796 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:09.796 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:33:09.797 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:09.797 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:09.798 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:09.798 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:09.799 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:33:09.799 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:09.799 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:09.800 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:33:09.800 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:09.801 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:09.801 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:09.802 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:33:09.802 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:09.803 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:09.803 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:09.803 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:09.804 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:33:09.804 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:09.805 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:09.805 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:33:09.806 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:09.806 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:09.806 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:09.807 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:33:09.807 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:09.807 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:09.808 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:09.808 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:33:09.808 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:09.809 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:33:09.809 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:33:09.809 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:33:09.810 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:09.810 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:33:09.811 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.825641_203.1 2021-11-11T21:33:14.812 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:14.813 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:14.813 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:33:14.872 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:33:14.873 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:33:15.046 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:33:15.047 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:33:15.047 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:33:15.047 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:33:15.048 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:33:15.048 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:33:15.048 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:15.048 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:15.048 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:33:15.049 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:15.049 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:33:15.049 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:15.049 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:15.050 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:33:15.050 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:15.050 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:33:15.050 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:15.050 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:33:15.051 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:15.051 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:15.051 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:15.051 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:15.052 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:33:15.052 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:15.052 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:15.052 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:33:15.052 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:15.053 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:15.053 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:15.053 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:33:15.053 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:15.054 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:15.054 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:15.054 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:15.054 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:33:15.054 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:15.055 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:15.055 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:33:15.055 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:15.056 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:15.056 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:15.056 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:33:15.056 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:15.057 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:15.057 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:15.057 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:15.057 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:33:15.057 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:15.058 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:15.058 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:33:15.058 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:15.058 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:15.058 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:15.059 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:33:15.059 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:15.059 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:15.059 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:15.060 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:33:15.060 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:15.060 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:33:15.060 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:33:15.060 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:33:15.061 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:15.061 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:33:15.061 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:33:20.062 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:20.063 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:20.064 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:33:20.148 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:33:20.148 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:33:20.621 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:33:20.621 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:33:20.621 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:33:20.622 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:33:20.622 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:33:20.622 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:33:20.622 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:20.623 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:20.623 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:33:20.623 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:20.623 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:33:20.623 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:20.624 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:20.624 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:33:20.624 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:20.624 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:33:20.624 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:20.625 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:33:20.625 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:20.625 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:20.625 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:20.626 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:20.626 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:33:20.626 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:20.626 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:20.627 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:33:20.627 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:20.627 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:20.627 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:20.627 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:33:20.628 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:20.628 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:20.628 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:20.628 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:20.629 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:33:20.629 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:20.629 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:20.629 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.914354_210.1", 2021-11-11T21:33:20.629 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:20.630 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:20.630 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:20.630 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.914354Z", 2021-11-11T21:33:20.630 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:20.631 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:20.631 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:20.631 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:20.631 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:33:20.632 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:20.632 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:20.632 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:33:20.632 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:20.633 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:20.633 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:20.633 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:33:20.633 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:20.633 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:20.634 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:20.634 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:33:20.634 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:20.634 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:33:20.635 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:33:20.635 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:33:20.635 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:20.635 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:33:20.637 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.914354_210.1 2021-11-11T21:33:25.638 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:25.639 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:25.639 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:33:25.695 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:33:25.695 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:33:26.102 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:33:26.102 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:33:26.103 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:33:26.103 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:33:26.103 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:33:26.103 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:33:26.104 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:26.104 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:26.104 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:33:26.104 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:26.104 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:33:26.105 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:26.105 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:26.105 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:33:26.105 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:26.106 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:33:26.106 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:26.106 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:33:26.106 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:26.107 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:26.107 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:26.107 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:26.107 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:33:26.108 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:26.108 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:26.108 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:33:26.108 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:26.108 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:26.109 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:26.109 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:33:26.109 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:26.109 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:26.110 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:26.110 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:26.110 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:33:26.110 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:26.111 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:26.111 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:33:26.111 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:26.111 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:26.112 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:26.112 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:33:26.112 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:26.112 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:26.112 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:26.113 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:26.113 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:33:26.113 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:26.113 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:26.113 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:33:26.114 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:26.114 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:26.114 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:26.114 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:33:26.115 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:26.115 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:26.115 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:26.115 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:33:26.116 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:26.116 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:33:26.116 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:33:26.116 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:33:26.117 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:26.117 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:33:26.119 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:33:31.120 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:31.121 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:31.122 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:33:31.169 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:33:31.170 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:33:31.242 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:33:31.243 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:33:31.243 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:33:31.243 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:33:31.244 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:33:31.244 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:33:31.244 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:31.244 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:31.245 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:33:31.245 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:31.245 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:33:31.245 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:31.245 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:31.246 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:33:31.246 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:31.246 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:33:31.246 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:31.247 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:33:31.247 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:31.247 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:31.247 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:31.248 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:31.248 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:33:31.248 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:31.248 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:31.248 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:33:31.249 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:31.249 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:31.249 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:31.249 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:33:31.250 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:31.250 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:31.250 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:31.250 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:31.251 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:33:31.251 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:31.251 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:31.251 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.977739_213.1", 2021-11-11T21:33:31.251 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:31.252 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:31.252 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:31.252 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.977739Z", 2021-11-11T21:33:31.252 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:31.253 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:31.253 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:31.253 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:31.253 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:33:31.254 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:31.254 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:31.254 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:33:31.254 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:31.255 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:31.255 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:31.255 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:33:31.255 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:31.256 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:31.256 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:31.256 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:33:31.256 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:31.257 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:33:31.257 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:33:31.257 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:33:31.257 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:31.258 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:33:31.259 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.977739_213.1 2021-11-11T21:33:36.260 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:36.260 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:36.261 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:33:36.540 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:33:36.541 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:33:36.617 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:33:36.617 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:33:36.617 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:33:36.618 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:33:36.618 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:33:36.618 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:33:36.618 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:36.619 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:36.619 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:33:36.619 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:36.619 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:33:36.620 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:36.620 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:36.620 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:33:36.620 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:36.620 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:33:36.621 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:36.621 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:33:36.621 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:36.621 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:36.622 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:36.622 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:36.622 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:33:36.622 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:36.623 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:36.623 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:33:36.623 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:36.623 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:36.624 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:36.624 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:33:36.624 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:36.624 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:36.624 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:36.625 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:36.625 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:33:36.625 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:36.625 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:36.626 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:33:36.626 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:36.626 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:36.627 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:36.627 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:33:36.627 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:36.627 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:36.627 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:36.628 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:36.628 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:33:36.628 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:36.628 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:36.629 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:33:36.629 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:36.629 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:36.629 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:36.629 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:33:36.630 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:36.630 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:36.630 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:36.630 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:33:36.631 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:36.631 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:33:36.631 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:33:36.631 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:33:36.631 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:36.632 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:33:36.633 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.787834_200.1 2021-11-11T21:33:41.634 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:41.635 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:41.636 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:33:41.692 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:33:41.692 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:33:41.773 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:33:41.774 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:33:41.774 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:33:41.774 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:33:41.774 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:33:41.775 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:33:41.775 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:41.775 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:41.775 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:33:41.776 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:41.776 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:33:41.776 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:41.776 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:41.777 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:33:41.777 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:41.777 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:33:41.777 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:41.778 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:33:41.778 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:41.778 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:41.778 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:41.779 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:41.779 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:33:41.779 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:41.779 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:41.779 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:33:41.780 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:41.780 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:41.780 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:41.780 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:33:41.781 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:41.781 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:41.781 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:41.781 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:41.781 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:33:41.782 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:41.782 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:41.782 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.794229_202.1", 2021-11-11T21:33:41.783 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:41.783 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:41.783 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:41.783 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.794229Z", 2021-11-11T21:33:41.784 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:41.784 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:41.784 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:41.784 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:41.785 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:33:41.785 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:41.785 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:41.785 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:33:41.786 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:41.786 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:41.786 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:41.786 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:33:41.787 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:41.787 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:41.787 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:41.787 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:33:41.787 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:41.788 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:33:41.788 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:33:41.788 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:33:41.788 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:41.789 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:33:41.789 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.794229_202.1 2021-11-11T21:33:46.790 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:46.791 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:46.792 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:33:46.850 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:33:46.850 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:33:46.937 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:33:46.937 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:33:46.938 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:33:46.938 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:33:46.938 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:33:46.938 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:33:46.938 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:46.939 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:46.939 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:33:46.939 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:46.939 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:33:46.940 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:46.940 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:46.940 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:33:46.940 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:46.940 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:33:46.941 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:46.941 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:33:46.941 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:46.941 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:46.942 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:46.942 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:46.942 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:33:46.942 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:46.942 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:46.943 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:33:46.943 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:46.943 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:46.943 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:46.944 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:33:46.944 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:46.944 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:46.944 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:46.944 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:46.945 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:33:46.945 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:46.945 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:46.945 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:33:46.945 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:46.946 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:46.946 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:46.946 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:33:46.946 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:46.947 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:46.947 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:46.947 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:46.947 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:33:46.947 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:46.948 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:46.948 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:33:46.948 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:46.948 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:46.948 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:46.949 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:33:46.949 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:46.949 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:46.949 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:46.950 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:33:46.950 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:46.950 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:33:46.950 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:33:46.950 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:33:46.951 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:46.951 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:33:46.952 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.768842_198.1 2021-11-11T21:33:51.953 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:51.954 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:51.955 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:33:52.005 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:33:52.005 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:33:52.082 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:33:52.083 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:33:52.083 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:33:52.083 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:33:52.084 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:33:52.084 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:33:52.084 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:52.084 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:52.084 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:33:52.085 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:52.085 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:33:52.085 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:52.085 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:52.085 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:33:52.086 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:52.086 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:33:52.086 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:52.086 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:33:52.086 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:52.087 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:52.087 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:52.087 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:52.087 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:33:52.088 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:52.088 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:52.088 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:33:52.088 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:52.089 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:52.089 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:52.089 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:33:52.089 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:52.089 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:52.090 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:52.090 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:52.090 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:33:52.090 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:52.090 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:52.091 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.080259_222.1", 2021-11-11T21:33:52.091 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:52.091 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:52.091 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:52.092 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.080259Z", 2021-11-11T21:33:52.092 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:52.092 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:52.092 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:52.092 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:52.093 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:33:52.093 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:52.093 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:52.093 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:33:52.093 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:52.094 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:52.094 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:52.094 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:33:52.094 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:52.095 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:52.095 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:52.095 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:33:52.095 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:52.095 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:33:52.096 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:33:52.096 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:33:52.096 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:52.096 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:33:52.098 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.080259_222.1 2021-11-11T21:33:57.099 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:57.100 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:33:57.101 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:33:57.143 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:33:57.143 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:33:57.218 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:33:57.218 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:33:57.218 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:33:57.218 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:33:57.219 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:33:57.219 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:33:57.219 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:57.220 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:57.220 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:33:57.220 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:57.220 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:33:57.220 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:57.221 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:57.221 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:33:57.221 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:57.221 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:33:57.222 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:57.222 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:33:57.222 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:57.222 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:57.223 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:57.223 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:57.223 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:33:57.223 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:57.223 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:57.224 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:33:57.224 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:57.224 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:57.224 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:57.225 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:33:57.225 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:57.225 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:57.225 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:57.226 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:57.226 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:33:57.226 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:57.226 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:57.227 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:33:57.227 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:57.227 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:57.227 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:57.227 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:33:57.228 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:57.228 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:57.228 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:57.228 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:33:57.229 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:33:57.229 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:33:57.229 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:33:57.229 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:33:57.230 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:33:57.230 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:33:57.230 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:33:57.230 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:33:57.231 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:33:57.231 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:57.231 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:57.231 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:33:57.232 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:33:57.232 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:33:57.232 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:33:57.232 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:33:57.232 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:33:57.233 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:33:57.234 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.831934_205.1 2021-11-11T21:34:02.236 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:02.236 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:02.237 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:34:02.311 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:34:02.312 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:34:02.687 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:34:02.688 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:34:02.688 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:34:02.689 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:34:02.689 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:34:02.689 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:34:02.690 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:02.690 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:02.690 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:34:02.690 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:02.691 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:34:02.691 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:02.691 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:02.691 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:34:02.692 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:02.692 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:34:02.692 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:02.692 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:34:02.693 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:02.693 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:02.693 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:02.693 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:02.694 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:34:02.694 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:02.694 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:02.694 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:34:02.695 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:02.695 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:02.695 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:02.695 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:34:02.696 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:02.696 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:02.696 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:02.696 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:02.697 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:34:02.697 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:02.697 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:02.697 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:34:02.698 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:02.698 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:02.698 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:02.698 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:34:02.698 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:02.699 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:02.699 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:02.699 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:02.699 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:34:02.700 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:02.700 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:02.700 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:34:02.700 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:02.701 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:02.701 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:02.701 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:34:02.701 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:02.701 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:02.702 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:02.702 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:34:02.702 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:02.702 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:34:02.703 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:34:02.703 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:34:02.703 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:02.703 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:34:02.705 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.830110_204.1 2021-11-11T21:34:07.706 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:07.707 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:07.708 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:34:07.752 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:34:07.752 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:34:07.824 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:34:07.824 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:34:07.824 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:34:07.825 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:34:07.825 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:34:07.825 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:34:07.825 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:07.825 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:07.826 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:34:07.826 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:07.826 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:34:07.826 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:07.827 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:07.827 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:34:07.827 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:07.827 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:34:07.828 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:07.828 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:34:07.828 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:07.828 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:07.828 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:07.829 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:07.829 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:34:07.829 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:07.829 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:07.830 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:34:07.830 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:07.830 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:07.830 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:07.830 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:34:07.831 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:07.831 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:07.831 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:07.831 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:07.832 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:34:07.832 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:07.832 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:07.832 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:34:07.833 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:07.833 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:07.833 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:07.833 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:34:07.834 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:07.834 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:07.834 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:07.834 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:07.835 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:34:07.835 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:07.835 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:07.835 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:34:07.835 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:07.836 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:07.836 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:07.836 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:34:07.836 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:07.837 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:07.837 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:07.837 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:34:07.837 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:07.837 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:34:07.838 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:34:07.838 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:34:07.838 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:07.838 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:34:07.840 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:34:12.841 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:12.841 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:12.842 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:34:12.889 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:34:12.890 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:34:12.966 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:34:12.966 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:34:12.966 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:34:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:34:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:34:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:34:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:12.968 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:12.968 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:34:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:34:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:34:12.970 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:12.970 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:34:12.970 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:12.970 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:34:12.971 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:12.971 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:12.971 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:12.971 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:12.972 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:34:12.972 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:12.972 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:12.972 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:34:12.973 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:12.973 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:12.973 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:12.973 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:34:12.973 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:12.974 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:12.974 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:12.974 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:12.974 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:34:12.975 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:12.975 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:12.975 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.977739_213.1", 2021-11-11T21:34:12.976 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:12.976 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:12.976 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:12.976 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.977739Z", 2021-11-11T21:34:12.977 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:12.977 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:12.977 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:12.977 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:12.978 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:34:12.978 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:12.978 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:12.978 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:34:12.978 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:12.979 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:12.979 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:12.979 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:34:12.979 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:12.980 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:12.980 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:12.980 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:34:12.980 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:12.981 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:34:12.981 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:34:12.981 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:34:12.981 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:12.981 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:34:12.983 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.977739_213.1 2021-11-11T21:34:17.984 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:17.985 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:17.985 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:34:18.037 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:34:18.037 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:34:18.112 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:34:18.112 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:34:18.112 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:34:18.113 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:34:18.113 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:34:18.113 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:34:18.113 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:18.113 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:18.114 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:34:18.114 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:18.114 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:34:18.114 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:18.115 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:18.115 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:34:18.115 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:18.115 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:34:18.116 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:18.116 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:34:18.116 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:18.116 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:18.117 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:18.117 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:18.117 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:34:18.117 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:18.118 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:18.118 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:34:18.118 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:18.118 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:18.118 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:18.119 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:34:18.119 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:18.119 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:18.119 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:18.120 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:18.120 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:34:18.120 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:18.120 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:18.121 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:34:18.121 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:18.121 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:18.121 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:18.121 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:34:18.122 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:18.122 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:18.122 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:18.122 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:18.123 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:34:18.123 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:18.123 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:18.123 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:34:18.123 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:18.124 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:18.124 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:18.124 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:34:18.124 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:18.125 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:18.125 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:18.125 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:34:18.125 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:18.125 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:34:18.126 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:34:18.126 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:34:18.126 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:18.126 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:34:18.127 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.787834_200.1 2021-11-11T21:34:23.129 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:23.130 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:23.131 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:34:23.208 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:34:23.208 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:34:23.346 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:34:23.346 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:34:23.347 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:34:23.347 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:34:23.347 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:34:23.347 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:34:23.348 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:23.348 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:23.348 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:34:23.348 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:23.348 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:34:23.349 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:23.349 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:23.349 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:34:23.349 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:23.350 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:34:23.350 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:23.350 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:34:23.350 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:23.351 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:23.351 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:23.351 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:23.352 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:34:23.352 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:23.352 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:23.353 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:34:23.353 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:23.353 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:23.353 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:23.353 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:34:23.354 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:23.354 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:23.354 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:23.354 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:23.355 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:34:23.355 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:23.355 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:23.355 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:34:23.356 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:23.356 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:23.356 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:23.356 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:34:23.357 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:23.357 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:23.357 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:23.357 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:23.358 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:34:23.358 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:23.358 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:23.358 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:34:23.359 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:23.359 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:23.359 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:23.359 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:34:23.359 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:23.360 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:23.360 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:23.360 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:34:23.360 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:23.361 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:34:23.361 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:34:23.361 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:34:23.362 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:23.362 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:34:23.363 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.831934_205.1 2021-11-11T21:34:28.364 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:28.365 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:28.366 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:34:28.427 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:34:28.427 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:34:28.570 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:34:28.570 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:34:28.571 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:34:28.571 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:34:28.571 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:34:28.571 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:34:28.572 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:28.572 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:28.572 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:34:28.572 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:28.573 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:34:28.573 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:28.573 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:28.573 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:34:28.573 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:28.574 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:34:28.574 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:28.574 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:34:28.574 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:28.575 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:28.575 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:28.575 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:28.575 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:34:28.575 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:28.576 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:28.576 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:34:28.576 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:28.576 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:28.577 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:28.577 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:34:28.577 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:28.577 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:28.577 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:28.578 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:28.578 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:34:28.578 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:28.578 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:28.579 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.977739_213.1", 2021-11-11T21:34:28.579 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:28.580 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:28.580 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:28.580 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.977739Z", 2021-11-11T21:34:28.581 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:28.581 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:28.581 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:28.581 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:28.582 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:34:28.582 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:28.582 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:28.582 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:34:28.583 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:28.583 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:28.583 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:28.583 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:34:28.584 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:28.584 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:28.584 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:28.584 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:34:28.585 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:28.585 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:34:28.585 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:34:28.585 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:34:28.586 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:28.586 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:34:28.586 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.977739_213.1 2021-11-11T21:34:33.587 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:33.588 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:33.589 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:34:33.637 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:34:33.637 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:34:34.120 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:34:34.121 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:34:34.121 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:34:34.121 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:34:34.121 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:34:34.122 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:34:34.122 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:34.122 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:34.122 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:34:34.123 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:34.123 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:34:34.123 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:34.123 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:34.124 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:34:34.124 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:34.124 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:34:34.124 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:34.125 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:34:34.125 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:34.125 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:34.125 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:34.125 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:34.126 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:34:34.126 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:34.126 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:34.126 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:34:34.127 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:34.127 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:34.127 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:34.127 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:34:34.128 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:34.128 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:34.128 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:34.128 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:34.128 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:34:34.129 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:34.129 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:34.129 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.977739_213.1", 2021-11-11T21:34:34.129 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:34.130 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:34.130 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:34.130 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.977739Z", 2021-11-11T21:34:34.130 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:34.131 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:34.131 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:34.131 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:34.131 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:34:34.132 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:34.132 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:34.132 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:34:34.132 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:34.133 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:34.133 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:34.133 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:34:34.133 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:34.133 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:34.134 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:34.134 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:34:34.134 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:34.134 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:34:34.135 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:34:34.135 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:34:34.135 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:34.135 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:34:34.137 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.977739_213.1 2021-11-11T21:34:39.137 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:39.138 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:39.139 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:34:39.201 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:34:39.201 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:34:39.272 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:34:39.272 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:34:39.272 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:34:39.273 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:34:39.273 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:34:39.274 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:34:39.274 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:39.274 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:39.275 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:34:39.275 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:39.276 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:34:39.276 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:39.276 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:39.277 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:34:39.277 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:39.277 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:34:39.277 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:39.278 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:34:39.278 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:39.278 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:39.278 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:39.278 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:39.279 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:34:39.279 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:39.279 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:39.279 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:34:39.280 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:39.280 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:39.280 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:39.280 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:34:39.281 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:39.281 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:39.281 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:39.282 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:39.282 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:34:39.283 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:39.283 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:39.284 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:34:39.284 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:39.284 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:39.285 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:39.285 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:34:39.286 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:39.286 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:39.287 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:39.287 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:39.287 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:34:39.288 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:39.288 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:39.289 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:34:39.289 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:39.289 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:39.290 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:39.290 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:34:39.290 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:39.291 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:39.291 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:39.292 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:34:39.292 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:39.292 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:34:39.293 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:34:39.293 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:34:39.293 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:39.294 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:34:39.296 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.787834_200.1 2021-11-11T21:34:44.297 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:44.297 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:44.298 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:34:44.345 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:34:44.345 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:34:44.417 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:34:44.418 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:34:44.418 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:34:44.418 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:34:44.418 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:34:44.419 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:34:44.419 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:44.419 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:44.419 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:34:44.420 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:44.420 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:34:44.420 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:44.420 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:44.420 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:34:44.421 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:44.421 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:34:44.421 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:44.421 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:34:44.422 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:44.422 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:44.422 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:44.422 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:44.423 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:34:44.423 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:44.423 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:44.423 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:34:44.423 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:44.424 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:44.424 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:44.424 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:34:44.424 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:44.425 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:44.425 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:44.425 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:44.425 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:34:44.425 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:44.426 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:44.426 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:34:44.426 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:44.426 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:44.427 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:44.427 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:34:44.427 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:44.427 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:44.427 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:44.428 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:44.428 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:34:44.428 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:44.428 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:44.429 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:34:44.429 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:44.429 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:44.429 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:44.429 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:34:44.430 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:44.430 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:44.430 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:44.431 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:34:44.431 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:44.431 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:34:44.431 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:34:44.432 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:34:44.432 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:44.432 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:34:44.434 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:34:49.435 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:49.436 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:49.437 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:34:49.509 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:34:49.509 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:34:49.763 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:34:49.763 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:34:49.763 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:34:49.764 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:34:49.764 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:34:49.764 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:34:49.764 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:49.765 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:49.765 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:34:49.765 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:49.765 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:34:49.766 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:49.766 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:49.766 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:34:49.766 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:49.766 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:34:49.767 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:49.767 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:34:49.767 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:49.767 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:49.768 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:49.768 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:49.768 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:34:49.768 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:49.769 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:49.769 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:34:49.769 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:49.769 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:49.770 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:49.770 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:34:49.770 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:49.770 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:49.770 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:49.771 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:49.771 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:34:49.771 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:49.772 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:49.772 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.794229_202.1", 2021-11-11T21:34:49.772 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:49.772 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:49.773 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:49.773 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.794229Z", 2021-11-11T21:34:49.773 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:49.774 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:49.774 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:49.774 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:49.774 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:34:49.775 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:49.775 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:49.775 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:34:49.775 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:49.776 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:49.776 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:49.776 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:34:49.776 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:49.777 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:49.777 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:49.777 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:34:49.777 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:49.777 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:34:49.778 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:34:49.778 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:34:49.778 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:49.778 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:34:49.780 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.794229_202.1 2021-11-11T21:34:54.781 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:54.782 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:34:54.783 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:34:54.842 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:34:54.842 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:34:55.045 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:34:55.045 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:34:55.045 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:34:55.046 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:34:55.046 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:34:55.046 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:34:55.047 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:55.047 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:55.047 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:34:55.047 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:55.048 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:34:55.048 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:55.048 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:55.048 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:34:55.049 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:55.049 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:34:55.049 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:55.049 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:34:55.049 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:55.050 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:55.050 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:55.050 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:55.051 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:34:55.052 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:55.052 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:55.052 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:34:55.052 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:55.053 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:55.053 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:55.053 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:34:55.053 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:55.054 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:55.054 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:55.054 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:55.054 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:34:55.055 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:55.055 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:55.055 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.050883_217.1", 2021-11-11T21:34:55.055 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:55.056 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:55.056 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:55.056 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.050883Z", 2021-11-11T21:34:55.056 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:55.056 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:55.057 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:55.057 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:34:55.057 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:34:55.057 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:34:55.058 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:34:55.058 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:34:55.058 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:34:55.058 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:34:55.058 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:34:55.059 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:34:55.059 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:34:55.059 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:55.059 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:55.060 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:34:55.060 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:34:55.060 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:34:55.060 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:34:55.060 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:34:55.061 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:34:55.061 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:34:55.062 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.050883_217.1 2021-11-11T21:35:00.064 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:00.065 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:00.065 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:00.146 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:00.147 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:00.664 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:00.665 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:00.665 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:00.665 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:00.665 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:00.666 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:00.666 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:00.666 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:00.666 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:00.667 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:00.667 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:00.667 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:00.667 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:00.668 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:00.668 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:00.668 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:00.668 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:00.668 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:00.669 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:00.669 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:00.669 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:00.670 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:00.670 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:00.670 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:00.671 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:00.671 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:00.671 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:00.671 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:00.672 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:00.672 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:00.672 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:00.673 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:00.673 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:00.673 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:00.673 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:00.674 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:00.674 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:00.674 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.794229_202.1", 2021-11-11T21:35:00.674 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:00.675 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:00.675 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:00.675 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.794229Z", 2021-11-11T21:35:00.676 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:00.676 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:00.676 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:00.676 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:00.677 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:00.677 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:00.677 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:00.678 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:00.678 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:00.678 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:00.678 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:00.678 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:00.679 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:00.679 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:00.679 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:00.679 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:00.680 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:00.680 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:00.680 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:00.681 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:00.681 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:00.681 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:00.682 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.794229_202.1 2021-11-11T21:35:05.682 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:05.683 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:05.684 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:05.728 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:05.728 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:05.803 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:05.804 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:05.804 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:05.804 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:05.804 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:05.805 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:05.805 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:05.805 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:05.805 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:05.805 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:05.806 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:05.806 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:05.806 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:05.806 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:05.807 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:05.807 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:05.807 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:05.807 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:05.807 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:05.808 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:05.808 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:05.808 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:05.808 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:05.808 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:05.809 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:05.809 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:05.809 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:05.809 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:05.810 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:05.810 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:05.810 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:05.810 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:05.811 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:05.811 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:05.811 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:05.811 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:05.812 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:05.812 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.117338_226.1", 2021-11-11T21:35:05.812 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:05.812 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:05.812 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:05.813 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.117338Z", 2021-11-11T21:35:05.813 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:05.813 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:05.813 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:05.814 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:05.814 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:05.814 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:05.814 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:05.814 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:05.815 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:05.815 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:05.815 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:05.816 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:05.816 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:05.816 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:05.816 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:05.816 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:05.817 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:05.817 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:05.817 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:05.817 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:05.818 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:05.818 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:05.819 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.117338_226.1 2021-11-11T21:35:10.820 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:10.821 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:10.822 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:10.871 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:10.872 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:10.953 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:10.954 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:10.954 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:10.954 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:10.955 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:10.955 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:10.955 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:10.955 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:10.956 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:10.956 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:10.956 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:10.956 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:10.957 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:10.957 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:10.957 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:10.957 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:10.957 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:10.958 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:10.958 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:10.958 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:10.958 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:10.959 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:10.959 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:10.959 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:10.960 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:10.960 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:10.960 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:10.960 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:10.961 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:10.961 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:10.961 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:10.961 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:10.962 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:10.962 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:10.962 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:10.963 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:10.963 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:10.963 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:35:10.963 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:10.963 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:10.964 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:10.964 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:35:10.964 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:10.964 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:10.965 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:10.965 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:10.965 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:10.965 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:10.965 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:10.966 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:10.966 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:10.966 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:10.966 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:10.967 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:10.967 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:10.967 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:10.967 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:10.967 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:10.968 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:10.968 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:10.968 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:10.968 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:10.969 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:10.969 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:10.970 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.831934_205.1 2021-11-11T21:35:15.972 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:15.972 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:15.973 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:16.015 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:16.016 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:16.141 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:16.142 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:16.142 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:16.142 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:16.142 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:16.143 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:16.143 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:16.143 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:16.143 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:16.144 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:16.144 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:16.144 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:16.144 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:16.144 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:16.145 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:16.145 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:16.145 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:16.145 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:16.146 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:16.146 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:16.146 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:16.146 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:16.147 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:16.147 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:16.147 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:16.147 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:16.148 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:16.148 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:16.148 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:16.148 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:16.148 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:16.149 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:16.149 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:16.149 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:16.149 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:16.150 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:16.150 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:16.150 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:35:16.150 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:16.151 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:16.151 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:16.151 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:35:16.151 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:16.152 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:16.152 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:16.152 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:16.152 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:16.152 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:16.153 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:16.153 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:16.153 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:16.153 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:16.154 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:16.154 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:16.154 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:16.154 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:16.155 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:16.155 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:16.155 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:16.155 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:16.156 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:16.156 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:16.156 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:16.156 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:16.157 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:35:21.158 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:21.159 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:21.160 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:21.218 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:21.219 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:21.740 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:21.741 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:21.741 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:21.741 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:21.742 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:21.742 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:21.742 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:21.742 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:21.743 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:21.743 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:21.743 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:21.743 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:21.743 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:21.744 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:21.744 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:21.744 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:21.744 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:21.745 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:21.745 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:21.745 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:21.745 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:21.746 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:21.746 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:21.746 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:21.746 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:21.746 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:21.747 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:21.747 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:21.747 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:21.747 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:21.748 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:21.748 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:21.748 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:21.748 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:21.748 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:21.749 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:21.749 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:21.749 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:35:21.749 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:21.750 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:21.750 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:21.750 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:35:21.750 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:21.750 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:21.751 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:21.751 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:21.751 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:21.751 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:21.751 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:21.752 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:21.752 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:21.752 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:21.752 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:21.753 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:21.753 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:21.753 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:21.753 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:21.753 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:21.754 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:21.754 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:21.754 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:21.754 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:21.755 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:21.755 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:21.756 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.768842_198.1 2021-11-11T21:35:26.757 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:26.758 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:26.759 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:26.806 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:26.807 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:26.885 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:26.886 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:26.886 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:26.886 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:26.887 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:26.887 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:26.887 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:26.887 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:26.887 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:26.888 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:26.888 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:26.888 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:26.888 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:26.889 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:26.889 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:26.889 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:26.890 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:26.890 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:26.891 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:26.891 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:26.891 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:26.892 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:26.892 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:26.893 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:26.893 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:26.893 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:26.894 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:26.894 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:26.894 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:26.894 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:26.895 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:26.895 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:26.895 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:26.895 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:26.895 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:26.896 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:26.896 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:26.896 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:35:26.896 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:26.897 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:26.897 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:26.897 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:35:26.897 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:26.897 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:26.898 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:26.898 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:26.898 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:26.899 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:26.899 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:26.899 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:26.899 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:26.899 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:26.900 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:26.900 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:26.900 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:26.900 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:26.901 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:26.901 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:26.901 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:26.901 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:26.901 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:26.902 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:26.902 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:26.902 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:26.903 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.783136_199.1 2021-11-11T21:35:31.905 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:31.906 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:31.906 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:31.951 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:31.951 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:32.019 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:32.019 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:32.020 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:32.020 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:32.020 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:32.020 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:32.021 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:32.021 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:32.021 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:32.021 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:32.022 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:32.022 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:32.022 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:32.022 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:32.023 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:32.023 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:32.023 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:32.024 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:32.024 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:32.024 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:32.024 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:32.025 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:32.025 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:32.025 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:32.025 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:32.025 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:32.026 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:32.026 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:32.026 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:32.026 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:32.027 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:32.027 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:32.027 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:32.027 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:32.028 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:32.028 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:32.028 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:32.028 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:35:32.029 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:32.029 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:32.029 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:32.029 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:35:32.030 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:32.030 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:32.030 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:32.030 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:32.030 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:32.031 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:32.031 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:32.031 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:32.031 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:32.032 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:32.032 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:32.032 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:32.032 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:32.033 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:32.033 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:32.033 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:32.033 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:32.033 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:32.034 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:32.034 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:32.034 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:32.034 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:32.036 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.787834_200.1 2021-11-11T21:35:37.037 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:37.038 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:37.039 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:37.127 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:37.128 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:37.364 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:37.365 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:37.365 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:37.365 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:37.365 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:37.366 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:37.366 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:37.366 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:37.366 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:37.367 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:37.367 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:37.367 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:37.367 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:37.368 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:37.368 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:37.368 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:37.368 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:37.368 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:37.369 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:37.369 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:37.369 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:37.369 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:37.370 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:37.370 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:37.370 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:37.370 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:37.371 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:37.371 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:37.372 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:37.373 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:37.373 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:37.374 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:37.374 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:37.375 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:37.375 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:37.376 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:37.376 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:37.376 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.055034_218.1", 2021-11-11T21:35:37.377 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:37.377 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:37.377 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:37.377 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.055034Z", 2021-11-11T21:35:37.377 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:37.378 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:37.378 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:37.378 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:37.378 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:37.379 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:37.379 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:37.379 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:37.379 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:37.380 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:37.380 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:37.380 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:37.381 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:37.381 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:37.381 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:37.381 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:37.381 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:37.382 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:37.382 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:37.382 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:37.383 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:37.383 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:37.385 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.055034_218.1 2021-11-11T21:35:42.387 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:42.388 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:42.389 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:42.431 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:42.432 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:42.508 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:42.509 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:42.509 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:42.509 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:42.509 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:42.510 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:42.510 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:42.510 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:42.510 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:42.510 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:42.511 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:42.511 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:42.511 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:42.511 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:42.512 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:42.512 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:42.512 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:42.512 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:42.513 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:42.513 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:42.513 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:42.513 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:42.513 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:42.514 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:42.514 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:42.514 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:42.514 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:42.515 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:42.515 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:42.515 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:42.515 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:42.516 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:42.516 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:42.516 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:42.516 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:42.517 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:42.517 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:42.517 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:35:42.517 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:42.517 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:42.518 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:42.518 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:35:42.518 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:42.518 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:42.519 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:42.519 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:42.519 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:42.519 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:42.519 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:42.520 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:42.520 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:42.520 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:42.520 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:42.521 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:42.521 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:42.521 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:42.521 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:42.521 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:42.522 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:42.522 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:42.522 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:42.522 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:42.523 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:42.523 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:42.524 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.783136_199.1 2021-11-11T21:35:47.526 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:47.527 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:47.527 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:47.589 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:47.590 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:48.087 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:48.088 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:48.088 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:48.088 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:48.089 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:48.089 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:48.089 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:48.089 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:48.090 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:48.090 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:48.090 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:48.090 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:48.091 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:48.091 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:48.091 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:48.091 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:48.091 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:48.092 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:48.092 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:48.092 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:48.092 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:48.093 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:48.093 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:48.093 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:48.093 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:48.093 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:48.094 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:48.094 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:48.094 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:48.094 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:48.095 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:48.095 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:48.095 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:48.095 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:48.096 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:48.096 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:48.096 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:48.096 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:35:48.096 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:48.097 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:48.097 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:48.097 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:35:48.097 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:48.098 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:48.098 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:48.098 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:48.098 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:48.098 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:48.099 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:48.099 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:48.099 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:48.099 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:48.100 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:48.100 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:48.100 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:48.100 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:48.100 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:48.101 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:48.101 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:48.101 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:48.101 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:48.101 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:48.102 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:48.102 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:48.102 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.783136_199.1 2021-11-11T21:35:53.104 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:53.105 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:53.106 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:53.155 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:53.155 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:53.240 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:53.240 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:53.241 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:53.241 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:53.241 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:53.241 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:53.242 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:53.242 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:53.242 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:53.242 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:53.243 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:53.243 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:53.243 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:53.243 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:53.243 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:53.244 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:53.244 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:53.244 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:53.244 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:53.245 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:53.245 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:53.245 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:53.245 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:53.245 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:53.246 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:53.246 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:53.246 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:53.246 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:53.247 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:53.247 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:53.247 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:53.247 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:53.247 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:53.248 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:53.248 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:53.248 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:53.248 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:53.249 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.863466_209.1", 2021-11-11T21:35:53.249 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:53.249 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:53.249 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:53.249 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.863466Z", 2021-11-11T21:35:53.250 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:53.250 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:53.250 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:53.250 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:53.251 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:53.251 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:53.251 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:53.251 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:53.251 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:53.252 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:53.252 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:53.252 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:53.252 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:53.252 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:53.253 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:53.253 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:53.253 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:53.253 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:53.254 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:53.254 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:53.254 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:53.254 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:53.256 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.863466_209.1 2021-11-11T21:35:58.256 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:58.257 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:35:58.258 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:35:58.308 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:35:58.308 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:35:58.385 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:35:58.386 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:35:58.386 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:35:58.386 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:35:58.386 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:35:58.387 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:35:58.387 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:58.387 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:58.387 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:35:58.388 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:58.388 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:35:58.388 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:58.388 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:58.388 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:35:58.389 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:58.389 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:35:58.389 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:58.389 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:35:58.390 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:58.390 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:58.390 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:58.390 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:58.390 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:35:58.391 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:58.391 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:58.391 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:35:58.391 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:58.392 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:58.392 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:58.392 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:35:58.392 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:58.392 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:58.393 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:58.393 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:58.393 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:35:58.393 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:58.394 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:58.394 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.977739_213.1", 2021-11-11T21:35:58.394 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:58.394 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:58.395 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:58.395 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.977739Z", 2021-11-11T21:35:58.395 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:58.395 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:58.395 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:58.396 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:35:58.396 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:35:58.396 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:35:58.396 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:35:58.396 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:35:58.397 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:35:58.397 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:35:58.397 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:35:58.397 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:35:58.398 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:35:58.398 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:58.398 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:58.398 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:35:58.399 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:35:58.399 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:35:58.399 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:35:58.399 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:35:58.399 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:35:58.400 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:35:58.401 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.977739_213.1 2021-11-11T21:36:03.402 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:03.404 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:03.405 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:03.496 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:03.497 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:03.833 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:36:03.833 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:36:03.834 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:36:03.834 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:36:03.834 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:36:03.834 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:36:03.835 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:03.835 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:03.835 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:36:03.835 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:03.835 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:36:03.836 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:03.836 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:03.836 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:03.836 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:03.837 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:36:03.837 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:03.837 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:36:03.837 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:03.838 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:03.838 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:03.838 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:03.838 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:36:03.839 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:03.839 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:03.839 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:36:03.839 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:03.840 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:03.840 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:03.840 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:36:03.840 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:03.840 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:03.841 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:03.841 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:03.841 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:36:03.841 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:03.841 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:03.842 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.863466_209.1", 2021-11-11T21:36:03.842 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:03.842 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:03.842 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:03.842 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.863466Z", 2021-11-11T21:36:03.843 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:03.843 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:03.844 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:03.844 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:03.844 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:36:03.844 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:03.845 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:03.845 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:03.845 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:03.845 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:03.846 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:03.846 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:36:03.847 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:03.847 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:03.848 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:03.848 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:36:03.848 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:03.849 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:36:03.849 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:36:03.850 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:36:03.850 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:03.851 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:36:03.853 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.863466_209.1 2021-11-11T21:36:08.855 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_multi_zone_redirect ... FAIL 2021-11-11T21:36:08.856 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:08.857 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:08.858 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:08.900 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:08.900 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:08.955 INFO:teuthology.orchestra.run.smithi096.stderr:No --period given, using current period=2aa9d15b-21de-40bc-a7d3-3d5e5933111b 2021-11-11T21:36:08.972 INFO:teuthology.orchestra.run.smithi096.stdout:[ 2021-11-11T21:36:08.972 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:36:08.973 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:08.973 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:09:40.616234Z" 2021-11-11T21:36:08.973 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:36:08.973 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:36:08.973 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "", 2021-11-11T21:36:08.974 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "0.000000" 2021-11-11T21:36:08.974 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:36:08.974 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:36:08.974 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636666253.051934_329.1", 2021-11-11T21:36:08.975 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:30:53.051934Z" 2021-11-11T21:36:08.975 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:36:08.975 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:36:08.975 INFO:teuthology.orchestra.run.smithi096.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:08.976 INFO:teuthology.orchestra.run.smithi096.stdout: "last_update": "2021-11-11T21:10:18.067753Z" 2021-11-11T21:36:08.976 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:36:08.976 INFO:teuthology.orchestra.run.smithi096.stdout:] 2021-11-11T21:36:08.977 INFO:rgw_multi.tests:starting meta checkpoint for zone=test-zone2 2021-11-11T21:36:08.978 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:08.978 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:08.979 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:09.064 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:09.065 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:09.306 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:36:09.307 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:36:09.307 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:36:09.307 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:36:09.308 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:36:09.308 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:36:09.308 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:09.309 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:09.309 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:36:09.309 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:09.309 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:36:09.310 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:09.310 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:09.310 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:09.310 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:09.310 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:36:09.311 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:09.311 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:36:09.312 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:09.313 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:09.313 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:09.313 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:09.314 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:36:09.314 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:09.314 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:09.314 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:36:09.315 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:09.315 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:09.315 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:09.315 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:36:09.315 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:09.316 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:09.316 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:09.317 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:09.317 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:36:09.318 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:09.318 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:09.318 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:36:09.319 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:09.319 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:09.319 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:09.319 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:36:09.319 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:09.320 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:09.320 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:09.320 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:09.320 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:36:09.321 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:09.321 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:09.321 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:09.321 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:09.321 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:09.322 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:09.322 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:36:09.322 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:09.322 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:09.323 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:09.323 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:36:09.323 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:09.323 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:36:09.324 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:36:09.324 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:36:09.324 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:09.324 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:36:09.326 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.783136_199.1 2021-11-11T21:36:14.327 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:14.328 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:14.329 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:14.372 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:14.372 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:14.438 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:36:14.439 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:36:14.439 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:36:14.439 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:36:14.440 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:36:14.440 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:36:14.440 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:14.440 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:14.441 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:36:14.441 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:14.441 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:36:14.441 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:14.441 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:14.442 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:14.442 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:14.442 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:36:14.442 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:14.443 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:36:14.443 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:14.443 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:14.443 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:14.444 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:14.444 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:36:14.444 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:14.444 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:14.445 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:36:14.445 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:14.445 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:14.445 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:14.445 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:36:14.446 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:14.446 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:14.446 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:14.446 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:14.447 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:36:14.447 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:14.447 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:14.447 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:36:14.448 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:14.448 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:14.448 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:14.448 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:36:14.449 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:14.449 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:14.449 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:14.449 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:14.449 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:36:14.450 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:14.450 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:14.450 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:14.450 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:14.450 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:14.451 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:14.451 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:36:14.451 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:14.451 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:14.452 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:14.452 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:36:14.452 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:14.452 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:36:14.453 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:36:14.453 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:36:14.453 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:14.453 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:36:14.455 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.831934_205.1 2021-11-11T21:36:19.456 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:19.456 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:19.457 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:19.506 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:19.507 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:19.589 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:36:19.589 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:36:19.589 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:36:19.590 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:36:19.590 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:36:19.590 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:36:19.590 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:19.591 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:19.591 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:36:19.591 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:19.591 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:36:19.592 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:19.592 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:19.592 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:19.592 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:19.592 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:36:19.593 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:19.593 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:36:19.593 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:19.593 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:19.594 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:19.594 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:19.594 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:36:19.594 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:19.595 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:19.595 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:36:19.595 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:19.595 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:19.596 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:19.596 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:36:19.596 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:19.596 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:19.596 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:19.597 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:19.597 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:36:19.597 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:19.597 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:19.598 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:36:19.598 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:19.598 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:19.598 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:19.599 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:36:19.599 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:19.599 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:19.599 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:19.599 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:19.600 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:36:19.600 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:19.600 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:19.600 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:19.600 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:19.601 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:19.601 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:19.601 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:36:19.601 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:19.602 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:19.602 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:19.602 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:36:19.602 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:19.603 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:36:19.603 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:36:19.603 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:36:19.603 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:19.603 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:36:19.605 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.825641_203.1 2021-11-11T21:36:24.606 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:24.607 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:24.607 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:24.702 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:24.702 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:24.876 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:36:24.877 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:36:24.877 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:36:24.877 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:36:24.877 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:36:24.878 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:36:24.878 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:24.878 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:24.878 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:36:24.879 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:24.879 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:36:24.879 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:24.879 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:24.880 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:24.880 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:24.880 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:36:24.880 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:24.881 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:36:24.881 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:24.881 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:24.881 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:24.881 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:24.882 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:36:24.882 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:24.882 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:24.882 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:36:24.883 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:24.883 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:24.883 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:24.883 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:36:24.884 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:24.884 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:24.884 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:24.884 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:24.884 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:36:24.885 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:24.885 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:24.885 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:36:24.885 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:24.886 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:24.886 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:24.886 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:36:24.886 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:24.887 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:24.887 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:24.887 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:24.887 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:36:24.888 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:24.888 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:24.888 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:24.888 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:24.889 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:24.889 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:24.889 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:36:24.889 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:24.889 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:24.890 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:24.890 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:36:24.890 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:24.890 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:36:24.891 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:36:24.891 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:36:24.891 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:24.891 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:36:24.893 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:36:29.893 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:29.894 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:29.895 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:29.959 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:29.960 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:30.473 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:36:30.474 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:36:30.474 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:36:30.474 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:36:30.475 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:36:30.475 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:36:30.475 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:30.475 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:30.476 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:36:30.476 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:30.476 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:36:30.476 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:30.477 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:30.477 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:30.477 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:30.477 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:36:30.477 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:30.478 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:36:30.478 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:30.478 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:30.478 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:30.479 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:30.479 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:36:30.479 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:30.480 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:30.480 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:36:30.480 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:30.481 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:30.481 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:30.481 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:36:30.481 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:30.482 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:30.482 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:30.482 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:30.483 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:36:30.483 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:30.483 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:30.483 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:36:30.484 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:30.484 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:30.484 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:30.484 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:36:30.485 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:30.485 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:30.485 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:30.485 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:30.485 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:36:30.486 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:30.486 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:30.486 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:30.486 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:30.487 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:30.487 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:30.487 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:36:30.487 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:30.487 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:30.488 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:30.488 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:36:30.488 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:30.488 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:36:30.489 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:36:30.489 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:36:30.489 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:30.489 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:36:30.491 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.825641_203.1 2021-11-11T21:36:35.491 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:35.492 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:35.493 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:35.595 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:35.595 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:35.842 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:36:35.843 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:36:35.843 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:36:35.844 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:36:35.844 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:36:35.844 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:36:35.845 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:35.845 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:35.845 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:36:35.845 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:35.846 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:36:35.846 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:35.846 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:35.846 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:35.846 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:35.847 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:36:35.847 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:35.847 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:36:35.847 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:35.848 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:35.848 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:35.848 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:35.848 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:36:35.849 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:35.849 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:35.849 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:36:35.849 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:35.849 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:35.850 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:35.850 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:36:35.850 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:35.850 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:35.851 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:35.851 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:35.851 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:36:35.851 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:35.852 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:35.852 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:36:35.852 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:35.852 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:35.853 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:35.853 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:36:35.853 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:35.853 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:35.854 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:35.854 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:35.854 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:36:35.854 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:35.854 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:35.855 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:35.855 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:35.855 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:35.855 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:35.856 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:36:35.856 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:35.856 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:35.856 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:35.856 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:36:35.857 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:35.857 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:36:35.857 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:36:35.857 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:36:35.857 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:35.858 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:36:35.859 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.830110_204.1 2021-11-11T21:36:40.860 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:40.861 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:40.862 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:40.908 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:40.909 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:40.985 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:36:40.985 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:36:40.985 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:36:40.986 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:36:40.986 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:36:40.986 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:36:40.986 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:40.987 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:40.987 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:36:40.987 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:40.987 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:36:40.988 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:40.988 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:40.988 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:40.988 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:40.988 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:36:40.989 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:40.989 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:36:40.989 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:40.990 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:40.990 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:40.990 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:40.990 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:36:40.991 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:40.991 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:40.991 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:36:40.991 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:40.991 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:40.992 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:40.992 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:36:40.992 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:40.992 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:40.993 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:40.993 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:40.993 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:36:40.993 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:40.993 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:40.994 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.080259_222.1", 2021-11-11T21:36:40.994 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:40.994 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:40.994 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:40.995 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.080259Z", 2021-11-11T21:36:40.995 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:40.995 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:40.995 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:40.995 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:40.996 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:36:40.996 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:40.996 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:40.996 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:40.997 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:40.997 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:40.997 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:40.997 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:36:40.997 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:40.998 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:40.998 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:40.998 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:36:40.998 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:40.999 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:36:40.999 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:36:40.999 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:36:40.999 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:41.001 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:36:41.002 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.080259_222.1 2021-11-11T21:36:46.003 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:46.004 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:46.005 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:46.047 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:46.047 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:46.120 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:36:46.120 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:36:46.120 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:36:46.121 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:36:46.121 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:36:46.121 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:36:46.121 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:46.121 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:46.122 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:36:46.122 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:46.122 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:36:46.122 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:46.123 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:46.123 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:46.123 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:46.123 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:36:46.124 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:46.124 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:36:46.124 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:46.124 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:46.125 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:46.125 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:46.125 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:36:46.125 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:46.125 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:46.126 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:36:46.126 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:46.126 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:46.126 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:46.127 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:36:46.127 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:46.127 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:46.127 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:46.128 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:46.128 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:36:46.128 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:46.128 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:46.128 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.863466_209.1", 2021-11-11T21:36:46.129 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:46.129 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:46.129 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:46.129 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.863466Z", 2021-11-11T21:36:46.130 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:46.130 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:46.130 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:46.130 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:46.130 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:36:46.131 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:46.131 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:46.131 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:46.131 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:46.132 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:46.132 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:46.132 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:36:46.132 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:46.132 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:46.133 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:46.133 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:36:46.133 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:46.133 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:36:46.134 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:36:46.134 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:36:46.134 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:46.134 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:36:46.136 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.863466_209.1 2021-11-11T21:36:51.137 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:51.138 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:51.139 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:51.213 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:51.214 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:51.441 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:36:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:36:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:36:51.442 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:36:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:36:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:36:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:51.443 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:36:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:36:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:51.444 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:51.445 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:51.445 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:51.445 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:36:51.445 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:51.446 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:36:51.446 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:51.446 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:51.446 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:51.446 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:51.447 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:36:51.447 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:51.447 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:51.447 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:36:51.448 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:51.448 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:51.448 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:51.448 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:36:51.449 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:51.449 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:51.449 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:51.449 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:51.449 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:36:51.450 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:51.450 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:51.450 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:36:51.450 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:51.451 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:51.451 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:51.451 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:36:51.451 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:51.452 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:51.452 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:51.452 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:51.452 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:36:51.453 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:51.453 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:51.453 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:51.453 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:51.454 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:51.454 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:51.454 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:36:51.454 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:51.454 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:51.455 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:51.455 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:36:51.455 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:51.455 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:36:51.456 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:36:51.456 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:36:51.456 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:51.456 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:36:51.458 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.783136_199.1 2021-11-11T21:36:56.459 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:56.460 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:36:56.461 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:36:56.507 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:36:56.508 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:36:56.589 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:36:56.589 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:36:56.590 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:36:56.590 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:36:56.590 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:36:56.591 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:36:56.591 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:56.591 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:56.591 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:36:56.592 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:56.592 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:36:56.592 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:56.592 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:56.592 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:36:56.593 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:56.593 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:36:56.593 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:56.593 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:36:56.593 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:56.594 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:56.594 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:56.594 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:56.594 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:36:56.595 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:56.595 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:56.595 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:36:56.595 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:56.596 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:56.596 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:56.597 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:36:56.598 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:56.598 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:56.598 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:56.598 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:56.599 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:36:56.599 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:56.599 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:56.599 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:36:56.600 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:56.600 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:56.600 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:56.600 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:36:56.600 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:56.601 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:56.601 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:56.601 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:36:56.601 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:36:56.602 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:36:56.602 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:36:56.602 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:36:56.602 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:36:56.603 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:36:56.603 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:36:56.603 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:36:56.603 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:36:56.603 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:56.604 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:56.604 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:36:56.604 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:36:56.604 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:36:56.605 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:36:56.605 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:36:56.605 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:36:56.605 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:36:56.607 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.783136_199.1 2021-11-11T21:37:01.608 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:01.609 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:01.610 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:37:01.658 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:37:01.658 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:37:01.739 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:37:01.739 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:37:01.740 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:37:01.740 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:37:01.740 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:37:01.740 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:37:01.741 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:01.741 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:01.741 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:37:01.741 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:01.742 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:37:01.742 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:01.742 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:01.742 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:37:01.742 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:01.743 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:37:01.743 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:01.743 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:37:01.743 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:01.744 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:01.744 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:01.744 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:01.744 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:37:01.745 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:01.745 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:01.745 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:37:01.745 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:01.745 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:01.746 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:01.746 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:37:01.746 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:01.746 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:01.747 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:01.747 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:01.747 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:37:01.747 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:01.748 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:01.748 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.049364_216.1", 2021-11-11T21:37:01.748 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:01.748 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:01.748 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:01.749 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.049364Z", 2021-11-11T21:37:01.749 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:01.749 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:01.749 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:01.749 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:01.750 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:37:01.750 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:01.750 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:01.750 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:37:01.751 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:01.751 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:01.751 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:01.751 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:37:01.752 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:01.752 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:01.752 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:01.753 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:37:01.753 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:01.753 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:37:01.754 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:37:01.754 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:37:01.754 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:01.754 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:37:01.756 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.049364_216.1 2021-11-11T21:37:06.757 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:06.757 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:06.758 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:37:06.801 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:37:06.802 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:37:06.867 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:37:06.867 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:37:06.868 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:37:06.868 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:37:06.868 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:37:06.868 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:37:06.869 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:06.869 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:06.869 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:37:06.869 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:06.870 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:37:06.870 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:06.870 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:06.870 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:37:06.870 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:06.871 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:37:06.871 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:06.871 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:37:06.871 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:06.872 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:06.872 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:06.872 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:06.872 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:37:06.873 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:06.873 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:06.873 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:37:06.873 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:06.874 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:06.874 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:06.874 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:37:06.874 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:06.874 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:06.875 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:06.875 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:06.875 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:37:06.875 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:06.876 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:06.876 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.917826_211.1", 2021-11-11T21:37:06.876 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:06.876 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:06.876 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:06.877 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.917826Z", 2021-11-11T21:37:06.877 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:06.877 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:06.877 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:06.878 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:06.878 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:37:06.878 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:06.878 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:06.879 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:37:06.879 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:06.879 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:06.879 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:06.879 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:37:06.880 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:06.880 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:06.880 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:06.880 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:37:06.881 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:06.881 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:37:06.881 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:37:06.881 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:37:06.881 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:06.882 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:37:06.883 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.917826_211.1 2021-11-11T21:37:11.884 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:11.885 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:11.886 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:37:11.981 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:37:11.981 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:37:12.433 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:37:12.434 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:37:12.434 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:37:12.435 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:37:12.435 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:37:12.435 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:37:12.436 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:12.436 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:12.436 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:37:12.436 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:12.437 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:37:12.437 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:12.437 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:12.437 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:37:12.438 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:12.438 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:37:12.438 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:12.438 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:37:12.438 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:12.439 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:12.439 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:12.439 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:12.439 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:37:12.440 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:12.440 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:12.440 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:37:12.440 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:12.440 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:12.441 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:12.441 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:37:12.441 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:12.441 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:12.442 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:12.442 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:12.442 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:37:12.442 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:12.442 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:12.443 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:37:12.443 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:12.443 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:12.443 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:12.444 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:37:12.444 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:12.444 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:12.445 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:12.445 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:12.445 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:37:12.446 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:12.446 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:12.446 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:37:12.446 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:12.446 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:12.447 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:12.447 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:37:12.447 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:12.447 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:12.448 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:12.448 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:37:12.448 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:12.448 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:37:12.449 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:37:12.449 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:37:12.449 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:12.449 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:37:12.451 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.768842_198.1 2021-11-11T21:37:17.452 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:17.454 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:17.455 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:37:17.535 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:37:17.535 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:37:17.645 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:37:17.646 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:37:17.646 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:37:17.647 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:37:17.647 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:37:17.647 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:37:17.647 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:17.648 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:17.648 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:37:17.648 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:17.648 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:37:17.649 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:17.649 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:17.649 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:37:17.650 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:17.650 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:37:17.650 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:17.650 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:37:17.651 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:17.651 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:17.651 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:17.651 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:17.652 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:37:17.652 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:17.652 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:17.652 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:37:17.653 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:17.653 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:17.653 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:17.653 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:37:17.654 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:17.654 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:17.654 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:17.654 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:17.655 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:37:17.655 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:17.655 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:17.656 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:37:17.656 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:17.656 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:17.656 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:17.658 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:37:17.658 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:17.658 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:17.659 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:17.659 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:17.659 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:37:17.659 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:17.660 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:17.660 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:37:17.660 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:17.660 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:17.661 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:17.661 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:37:17.661 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:17.661 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:17.661 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:17.662 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:37:17.662 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:17.662 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:37:17.662 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:37:17.663 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:37:17.663 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:17.663 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:37:17.665 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.768842_198.1 2021-11-11T21:37:22.665 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:22.666 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:22.667 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:37:22.736 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:37:22.737 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:37:23.313 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:37:23.314 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:37:23.314 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:37:23.314 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:37:23.314 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:37:23.315 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:37:23.315 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:23.315 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:23.315 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:37:23.315 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:23.316 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:37:23.316 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:23.316 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:23.316 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:37:23.317 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:23.317 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:37:23.317 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:23.317 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:37:23.317 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:23.318 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:23.318 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:23.318 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:23.319 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:37:23.319 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:23.319 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:23.319 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:37:23.320 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:23.320 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:23.320 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:23.320 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:37:23.320 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:23.321 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:23.321 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:23.321 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:23.321 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:37:23.321 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:23.322 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:23.322 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.109435_223.1", 2021-11-11T21:37:23.322 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:23.322 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:23.323 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:23.323 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.109435Z", 2021-11-11T21:37:23.323 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:23.323 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:23.323 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:23.324 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:23.324 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:37:23.324 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:23.324 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:23.325 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:37:23.325 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:23.325 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:23.325 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:23.325 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:37:23.326 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:23.326 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:23.326 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:23.326 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:37:23.327 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:23.327 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:37:23.327 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:37:23.327 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:37:23.327 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:23.328 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:37:23.329 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.109435_223.1 2021-11-11T21:37:28.330 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:28.331 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:28.332 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:37:28.380 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:37:28.380 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:37:28.458 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:37:28.458 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:37:28.458 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:37:28.459 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:37:28.459 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:37:28.459 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:37:28.459 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:28.460 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:28.460 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:37:28.460 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:28.460 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:37:28.461 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:28.461 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:28.461 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:37:28.461 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:28.462 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:37:28.462 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:28.462 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:37:28.462 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:28.463 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:28.463 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:28.463 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:28.463 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:37:28.464 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:28.464 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:28.464 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:37:28.464 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:28.465 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:28.465 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:28.465 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:37:28.465 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:28.466 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:28.466 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:28.466 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:28.466 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:37:28.466 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:28.467 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:28.467 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.794229_202.1", 2021-11-11T21:37:28.467 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:28.467 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:28.468 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:28.468 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.794229Z", 2021-11-11T21:37:28.468 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:28.468 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:28.469 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:28.469 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:28.469 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:37:28.469 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:28.469 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:28.470 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:37:28.470 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:28.470 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:28.470 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:28.471 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:37:28.471 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:28.471 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:28.471 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:28.472 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:37:28.472 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:28.472 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:37:28.472 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:37:28.472 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:37:28.473 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:28.473 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:37:28.474 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.794229_202.1 2021-11-11T21:37:33.476 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:33.476 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:33.477 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:37:33.528 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:37:33.528 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:37:33.606 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:37:33.607 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:37:33.607 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:37:33.607 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:37:33.608 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:37:33.608 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:37:33.608 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:33.608 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:33.609 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:37:33.609 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:33.609 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:37:33.609 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:33.609 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:33.610 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:37:33.610 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:33.610 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:37:33.611 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:33.611 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:37:33.611 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:33.611 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:33.612 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:33.612 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:33.612 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:37:33.613 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:33.613 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:33.613 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:37:33.614 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:33.614 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:33.614 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:33.614 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:37:33.615 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:33.615 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:33.615 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:33.615 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:33.615 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:37:33.616 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:33.616 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:33.616 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:37:33.616 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:33.616 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:33.617 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:33.617 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:37:33.617 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:33.617 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:33.618 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:33.618 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:33.618 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:37:33.618 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:33.618 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:33.619 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:37:33.619 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:33.619 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:33.619 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:33.619 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:37:33.620 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:33.620 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:33.620 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:33.620 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:37:33.621 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:33.621 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:37:33.621 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:37:33.621 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:37:33.622 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:33.622 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:37:33.623 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.768842_198.1 2021-11-11T21:37:38.624 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:38.625 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:38.626 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:37:38.700 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:37:38.700 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:37:38.815 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:37:38.816 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:37:38.816 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:37:38.816 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:37:38.817 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:37:38.817 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:37:38.817 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:38.817 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:38.818 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:37:38.818 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:38.818 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:37:38.818 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:38.818 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:38.819 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:37:38.819 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:38.819 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:37:38.819 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:38.820 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:37:38.820 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:38.820 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:38.820 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:38.821 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:38.821 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:37:38.821 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:38.821 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:38.821 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:37:38.822 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:38.822 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:38.822 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:38.822 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:37:38.823 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:38.823 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:38.823 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:38.823 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:38.824 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:37:38.824 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:38.824 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:38.824 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.917826_211.1", 2021-11-11T21:37:38.824 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:38.825 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:38.825 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:38.825 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.917826Z", 2021-11-11T21:37:38.825 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:38.826 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:38.826 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:38.826 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:38.826 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:37:38.826 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:38.827 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:38.827 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:37:38.827 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:38.827 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:38.828 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:38.828 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:37:38.828 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:38.828 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:38.829 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:38.829 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:37:38.829 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:38.829 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:37:38.829 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:37:38.830 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:37:38.830 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:38.830 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:37:38.831 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.917826_211.1 2021-11-11T21:37:43.833 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:43.834 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:43.834 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:37:43.898 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:37:43.899 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:37:44.316 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:37:44.317 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:37:44.317 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:37:44.317 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:37:44.318 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:37:44.318 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:37:44.318 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:44.318 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:44.319 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:37:44.319 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:44.319 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:37:44.319 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:44.320 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:44.320 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:37:44.320 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:44.320 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:37:44.321 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:44.321 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:37:44.321 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:44.321 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:44.321 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:44.322 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:44.322 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:37:44.322 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:44.322 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:44.323 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:37:44.323 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:44.323 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:44.323 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:44.323 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:37:44.324 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:44.324 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:44.324 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:44.324 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:44.324 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:37:44.325 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:44.325 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:44.325 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:37:44.325 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:44.326 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:44.326 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:44.326 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:37:44.326 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:44.326 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:44.327 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:44.327 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:44.327 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:37:44.327 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:44.328 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:44.328 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:37:44.328 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:44.328 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:44.328 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:44.329 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:37:44.329 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:44.329 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:44.329 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:44.330 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:37:44.330 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:44.330 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:37:44.330 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:37:44.330 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:37:44.331 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:44.331 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:37:44.332 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.787834_200.1 2021-11-11T21:37:49.333 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:49.334 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:49.335 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:37:49.405 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:37:49.405 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:37:49.505 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:37:49.505 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:37:49.505 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:37:49.506 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:37:49.506 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:37:49.506 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:37:49.506 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:49.507 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:49.507 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:37:49.507 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:49.507 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:37:49.508 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:49.508 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:49.508 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:37:49.508 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:49.509 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:37:49.509 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:49.509 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:37:49.509 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:49.510 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:49.510 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:49.510 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:49.510 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:37:49.510 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:49.511 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:49.511 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:37:49.511 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:49.511 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:49.512 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:49.512 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:37:49.512 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:49.512 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:49.513 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:49.513 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:49.513 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:37:49.513 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:49.514 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:49.514 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.783136_199.1", 2021-11-11T21:37:49.514 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:49.514 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:49.515 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:49.515 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.783136Z", 2021-11-11T21:37:49.515 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:49.515 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:49.515 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:49.516 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:49.516 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:37:49.516 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:49.516 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:49.517 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:37:49.517 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:49.517 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:49.517 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:49.517 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:37:49.518 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:49.518 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:49.518 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:49.518 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:37:49.519 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:49.519 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:37:49.519 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:37:49.519 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:37:49.519 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:49.520 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:37:49.521 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.783136_199.1 2021-11-11T21:37:54.523 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:54.524 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:37:54.524 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:37:54.622 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:37:54.622 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:37:54.995 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:37:54.995 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:37:54.995 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:37:54.996 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:37:54.996 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:37:54.996 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:37:54.996 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:54.997 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:54.997 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:37:54.997 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:54.997 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:37:54.998 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:54.998 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:54.998 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:37:54.998 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:54.999 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:37:54.999 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:54.999 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:37:54.999 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:55.000 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:55.000 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:55.000 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:55.000 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:37:55.001 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:55.001 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:55.001 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:37:55.001 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:55.002 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:55.002 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:55.002 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:37:55.002 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:55.003 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:55.003 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:55.003 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:55.003 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:37:55.004 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:55.004 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:55.004 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:37:55.004 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:55.004 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:55.005 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:55.005 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:37:55.005 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:55.005 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:55.006 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:55.006 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:37:55.006 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:37:55.006 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:37:55.006 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:37:55.007 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:37:55.007 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:37:55.007 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:37:55.007 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:37:55.008 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:37:55.008 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:37:55.008 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:55.008 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:55.008 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:37:55.009 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:37:55.009 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:37:55.009 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:37:55.009 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:37:55.010 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:37:55.010 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:37:55.011 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.830110_204.1 2021-11-11T21:38:00.013 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:00.014 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:00.014 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:00.081 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:00.081 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:00.573 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:00.574 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:00.574 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:00.575 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:00.575 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:00.575 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:00.576 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:00.576 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:00.576 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:00.577 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:00.577 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:00.577 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:00.577 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:00.578 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:00.578 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:00.578 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:00.579 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:00.579 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:00.580 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:00.580 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:00.580 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:00.581 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:00.581 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:00.582 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:00.582 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:00.582 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:00.583 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:00.583 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:00.583 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:00.584 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:00.584 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:00.584 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:00.585 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:00.585 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:00.586 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:00.586 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:00.586 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:00.586 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.973566_212.1", 2021-11-11T21:38:00.587 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:00.587 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:00.587 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:00.588 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.973566Z", 2021-11-11T21:38:00.588 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:00.588 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:00.588 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:00.589 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:00.589 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:00.589 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:00.590 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:00.590 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:00.590 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:00.591 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:00.591 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:00.591 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:00.592 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:00.592 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:00.592 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:00.592 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:00.593 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:00.593 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:00.593 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:00.594 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:00.594 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:00.594 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:00.595 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.973566_212.1 2021-11-11T21:38:05.597 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:05.597 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:05.598 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:05.679 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:05.680 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:06.244 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:06.244 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:06.244 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:06.245 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:06.245 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:06.245 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:06.245 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:06.246 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:06.246 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:06.246 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:06.246 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:06.247 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:06.247 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:06.247 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:06.247 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:06.248 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:06.248 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:06.248 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:06.248 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:06.248 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:06.249 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:06.249 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:06.249 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:06.249 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:06.250 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:06.250 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:06.250 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:06.250 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:06.251 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:06.251 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:06.251 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:06.251 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:06.251 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:06.252 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:06.252 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:06.252 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:06.252 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:06.253 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.055034_218.1", 2021-11-11T21:38:06.253 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:06.253 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:06.253 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:06.253 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.055034Z", 2021-11-11T21:38:06.254 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:06.254 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:06.254 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:06.254 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:06.255 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:06.255 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:06.255 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:06.255 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:06.256 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:06.256 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:06.256 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:06.256 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:06.256 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:06.257 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:06.257 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:06.257 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:06.257 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:06.258 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:06.258 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:06.258 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:06.258 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:06.259 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:06.260 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.055034_218.1 2021-11-11T21:38:11.261 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:11.262 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:11.262 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:11.323 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:11.324 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:11.622 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:11.622 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:11.623 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:11.623 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:11.623 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:11.623 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:11.624 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:11.624 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:11.624 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:11.624 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:11.624 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:11.625 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:11.625 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:11.625 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:11.625 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:11.626 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:11.626 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:11.626 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:11.626 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:11.626 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:11.627 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:11.627 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:11.627 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:11.627 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:11.628 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:11.628 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:11.628 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:11.628 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:11.628 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:11.629 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:11.629 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:11.630 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:11.630 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:11.630 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:11.630 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:11.630 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:11.631 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:11.631 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.044532_215.1", 2021-11-11T21:38:11.631 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:11.631 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:11.631 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:11.632 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.044532Z", 2021-11-11T21:38:11.632 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:11.632 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:11.632 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:11.633 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:11.633 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:11.633 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:11.633 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:11.633 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:11.634 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:11.634 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:11.634 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:11.634 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:11.635 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:11.635 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:11.635 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:11.635 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:11.635 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:11.636 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:11.636 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:11.636 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:11.636 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:11.636 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:11.637 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.044532_215.1 2021-11-11T21:38:16.638 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:16.639 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:16.640 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:16.690 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:16.690 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:16.767 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:16.767 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:16.768 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:16.768 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:16.768 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:16.768 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:16.768 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:16.769 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:16.769 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:16.769 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:16.769 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:16.770 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:16.770 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:16.770 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:16.770 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:16.771 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:16.771 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:16.772 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:16.772 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:16.772 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:16.773 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:16.773 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:16.773 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:16.773 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:16.774 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:16.774 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:16.774 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:16.774 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:16.775 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:16.775 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:16.775 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:16.775 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:16.776 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:16.776 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:16.776 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:16.776 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:16.776 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:16.777 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.838825_206.1", 2021-11-11T21:38:16.777 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:16.777 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:16.778 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:16.778 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.838825Z", 2021-11-11T21:38:16.778 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:16.778 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:16.779 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:16.779 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:16.779 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:16.779 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:16.780 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:16.780 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:16.780 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:16.780 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:16.780 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:16.781 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:16.781 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:16.781 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:16.781 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:16.781 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:16.782 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:16.782 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:16.782 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:16.782 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:16.783 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:16.783 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:16.784 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.838825_206.1 2021-11-11T21:38:21.786 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:21.787 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:21.788 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:21.847 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:21.847 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:21.986 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:21.986 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:21.986 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:21.987 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:21.987 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:21.987 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:21.987 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:21.988 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:21.988 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:21.988 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:21.988 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:21.989 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:21.989 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:21.989 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:21.989 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:21.990 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:21.990 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:21.990 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:21.990 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:21.990 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:21.991 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:21.991 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:21.991 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:21.992 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:21.992 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:21.992 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:21.992 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:21.993 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:21.993 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:21.993 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:21.993 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:21.994 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:21.994 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:21.994 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:21.994 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:21.995 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:21.995 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:21.995 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.049364_216.1", 2021-11-11T21:38:21.995 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:21.995 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:21.996 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:21.996 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.049364Z", 2021-11-11T21:38:21.996 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:21.996 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:21.997 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:21.997 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:21.997 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:21.997 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:21.998 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:21.998 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:21.998 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:21.998 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:21.998 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:21.999 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:21.999 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:21.999 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:21.999 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:21.999 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:22.000 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:22.000 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:22.000 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:22.000 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:22.001 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:22.001 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:22.002 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.049364_216.1 2021-11-11T21:38:27.004 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:27.005 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:27.005 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:27.090 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:27.090 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:27.181 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:27.181 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:27.182 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:27.182 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:27.182 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:27.182 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:27.183 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:27.183 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:27.183 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:27.183 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:27.184 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:27.184 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:27.184 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:27.184 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:27.185 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:27.185 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:27.185 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:27.185 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:27.185 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:27.186 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:27.186 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:27.186 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:27.186 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:27.187 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:27.187 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:27.187 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:27.187 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:27.188 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:27.188 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:27.188 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:27.188 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:27.188 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:27.189 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:27.189 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:27.189 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:27.189 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:27.190 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:27.190 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:38:27.190 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:27.190 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:27.191 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:27.191 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:38:27.191 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:27.191 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:27.192 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:27.192 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:27.192 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:27.192 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:27.192 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:27.193 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:27.193 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:27.193 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:27.193 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:27.194 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:27.194 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:27.194 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:27.194 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:27.194 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:27.195 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:27.195 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:27.195 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:27.195 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:27.196 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:27.196 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:27.197 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.831934_205.1 2021-11-11T21:38:32.198 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:32.199 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:32.199 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:32.249 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:32.250 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:32.336 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:32.336 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:32.337 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:32.337 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:32.337 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:32.337 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:32.338 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:32.338 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:32.338 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:32.338 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:32.339 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:32.339 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:32.339 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:32.339 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:32.339 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:32.340 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:32.340 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:32.340 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:32.340 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:32.341 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:32.341 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:32.341 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:32.341 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:32.341 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:32.342 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:32.342 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:32.342 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:32.342 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:32.343 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:32.343 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:32.343 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:32.343 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:32.343 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:32.344 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:32.344 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:32.344 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:32.344 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:32.345 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.794229_202.1", 2021-11-11T21:38:32.345 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:32.345 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:32.345 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:32.346 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.794229Z", 2021-11-11T21:38:32.346 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:32.346 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:32.347 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:32.347 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:32.347 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:32.348 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:32.348 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:32.348 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:32.348 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:32.348 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:32.349 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:32.349 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:32.349 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:32.350 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:32.350 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:32.350 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:32.351 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:32.351 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:32.351 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:32.351 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:32.351 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:32.352 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:32.353 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.794229_202.1 2021-11-11T21:38:37.354 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:37.355 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:37.355 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:37.398 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:37.399 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:37.468 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:37.468 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:37.468 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:37.469 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:37.469 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:37.469 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:37.469 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:37.470 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:37.470 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:37.470 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:37.471 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:37.471 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:37.471 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:37.471 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:37.471 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:37.472 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:37.472 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:37.472 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:37.472 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:37.473 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:37.473 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:37.473 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:37.474 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:37.474 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:37.474 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:37.474 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:37.475 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:37.475 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:37.475 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:37.475 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:37.475 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:37.476 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:37.476 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:37.476 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:37.476 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:37.477 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:37.477 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:37.477 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:38:37.477 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:37.478 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:37.478 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:37.478 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:38:37.478 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:37.479 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:37.479 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:37.479 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:37.479 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:37.480 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:37.480 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:37.480 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:37.480 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:37.480 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:37.481 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:37.481 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:37.481 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:37.481 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:37.482 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:37.482 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:37.482 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:37.483 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:37.483 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:37.483 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:37.484 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:37.484 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:37.485 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.787834_200.1 2021-11-11T21:38:42.486 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:42.487 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:42.488 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:42.572 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:42.572 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:42.819 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:42.820 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:42.820 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:42.820 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:42.820 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:42.821 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:42.821 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:42.821 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:42.821 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:42.822 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:42.822 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:42.822 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:42.822 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:42.822 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:42.823 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:42.823 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:42.823 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:42.823 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:42.824 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:42.824 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:42.824 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:42.825 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:42.825 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:42.825 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:42.825 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:42.826 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:42.826 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:42.826 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:42.826 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:42.827 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:42.827 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:42.827 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:42.827 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:42.829 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:42.829 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:42.829 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:42.830 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:42.830 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:38:42.830 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:42.830 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:42.831 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:42.831 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:38:42.831 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:42.831 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:42.832 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:42.832 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:42.832 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:42.832 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:42.833 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:42.833 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:42.833 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:42.833 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:42.833 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:42.834 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:42.834 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:42.834 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:42.834 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:42.834 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:42.835 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:42.835 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:42.835 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:42.835 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:42.836 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:42.836 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:42.837 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.789745_201.1 2021-11-11T21:38:47.839 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:47.839 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:47.840 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:47.905 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:47.906 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:48.317 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:48.317 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:48.318 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:48.318 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:48.318 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:48.318 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:48.319 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:48.319 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:48.319 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:48.319 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:48.320 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:48.320 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:48.320 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:48.320 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:48.321 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:48.321 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:48.321 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:48.321 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:48.322 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:48.322 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:48.322 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:48.322 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:48.322 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:48.323 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:48.323 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:48.323 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:48.323 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:48.324 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:48.324 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:48.324 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:48.324 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:48.324 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:48.325 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:48.325 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:48.325 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:48.325 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:48.325 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:48.326 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:38:48.326 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:48.326 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:48.326 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:48.327 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:38:48.327 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:48.327 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:48.327 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:48.327 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:48.328 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:48.328 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:48.328 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:48.328 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:48.329 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:48.329 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:48.329 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:48.329 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:48.330 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:48.330 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:48.330 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:48.330 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:48.331 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:48.331 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:48.331 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:48.331 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:48.331 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:48.332 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:48.333 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:38:53.335 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:53.335 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:53.336 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:53.423 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:53.423 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:53.720 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:53.721 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:53.721 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:53.721 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:53.721 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:53.722 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:53.722 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:53.722 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:53.722 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:53.723 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:53.723 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:53.723 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:53.723 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:53.723 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:53.724 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:53.724 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:53.724 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:53.724 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:53.725 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:53.725 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:53.725 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:53.725 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:53.725 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:53.726 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:53.726 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:53.726 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:53.726 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:53.727 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:53.727 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:53.727 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:53.727 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:53.727 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:53.728 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:53.728 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:53.728 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:53.728 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:53.728 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:53.729 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:38:53.729 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:53.729 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:53.729 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:53.730 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:38:53.730 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:53.730 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:53.730 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:53.730 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:53.731 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:53.731 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:53.731 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:53.731 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:53.731 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:53.732 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:53.732 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:53.732 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:53.732 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:53.732 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:53.733 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:53.733 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:53.733 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:53.733 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:53.733 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:53.734 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:53.734 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:53.734 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:53.735 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.825641_203.1 2021-11-11T21:38:58.736 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:58.737 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:38:58.737 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:38:58.784 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:38:58.784 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:38:58.855 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:38:58.855 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:38:58.855 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:38:58.856 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:38:58.856 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:38:58.856 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:38:58.856 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:58.856 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:58.857 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:38:58.857 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:58.857 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:38:58.857 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:58.858 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:58.858 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:38:58.858 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:58.858 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:38:58.859 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:58.859 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:38:58.859 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:58.859 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:58.860 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:58.860 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:58.860 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:38:58.860 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:58.861 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:58.861 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:38:58.861 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:58.861 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:58.861 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:58.862 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:38:58.862 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:58.862 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:58.863 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:58.863 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:58.863 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:38:58.863 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:58.864 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:58.864 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:38:58.864 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:58.864 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:58.864 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:58.865 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:38:58.865 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:58.865 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:58.865 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:58.866 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:38:58.866 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:38:58.866 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:38:58.866 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:38:58.867 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:38:58.867 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:38:58.867 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:38:58.867 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:38:58.868 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:38:58.868 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:38:58.868 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:58.869 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:58.869 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:38:58.869 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:38:58.870 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:38:58.870 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:38:58.870 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:38:58.870 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:38:58.871 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:38:58.872 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.768842_198.1 2021-11-11T21:39:03.873 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:03.874 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:03.874 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:39:03.917 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:39:03.917 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:39:03.982 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:39:03.983 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:39:03.983 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:39:03.983 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:39:03.983 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:39:03.984 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:39:03.984 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:03.984 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:03.984 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:39:03.984 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:03.985 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:39:03.985 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:03.985 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:03.985 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:39:03.986 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:03.986 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:39:03.986 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:03.986 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:39:03.987 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:03.987 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:03.987 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:03.987 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:03.988 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:39:03.988 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:03.988 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:03.989 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:39:03.989 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:03.989 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:03.989 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:03.990 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:39:03.990 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:03.990 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:03.990 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:03.991 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:03.991 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:39:03.991 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:03.991 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:03.992 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:39:03.992 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:03.992 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:03.992 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:03.993 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:39:03.993 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:03.993 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:03.993 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:03.993 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:03.994 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:39:03.994 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:03.994 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:03.994 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:39:03.995 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:03.995 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:03.995 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:03.995 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:39:03.996 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:03.996 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:03.996 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:03.996 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:39:03.996 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:03.997 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:39:03.997 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:39:03.997 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:39:03.997 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:03.998 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:39:03.999 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:39:09.000 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:09.001 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:09.002 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:39:09.086 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:39:09.087 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:39:09.276 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:39:09.276 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:39:09.277 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:39:09.277 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:39:09.277 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:39:09.277 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:39:09.277 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:09.278 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:09.278 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:39:09.278 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:09.278 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:39:09.279 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:09.279 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:09.279 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:39:09.279 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:09.279 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:39:09.280 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:09.280 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:39:09.280 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:09.280 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:09.281 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:09.281 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:09.281 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:39:09.281 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:09.282 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:09.282 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:39:09.282 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:09.282 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:09.282 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:09.283 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:39:09.283 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:09.283 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:09.283 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:09.284 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:09.284 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:39:09.284 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:09.284 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:09.284 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:39:09.285 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:09.285 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:09.285 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:09.285 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:39:09.286 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:09.286 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:09.286 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:09.286 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:09.286 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:39:09.287 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:09.287 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:09.287 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:39:09.287 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:09.288 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:09.288 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:09.288 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:39:09.288 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:09.288 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:09.289 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:09.289 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:39:09.289 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:09.289 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:39:09.290 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:39:09.290 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:39:09.290 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:09.290 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:39:09.298 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.768842_198.1 2021-11-11T21:39:14.300 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:14.300 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:14.301 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:39:14.345 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:39:14.345 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:39:14.480 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:39:14.481 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:39:14.481 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:39:14.481 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:39:14.482 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:39:14.482 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:39:14.482 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:14.482 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:14.483 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:39:14.483 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:14.483 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:39:14.483 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:14.484 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:14.484 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:39:14.484 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:14.484 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:39:14.485 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:14.485 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:39:14.485 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:14.485 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:14.486 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:14.486 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:14.486 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:39:14.486 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:14.487 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:14.487 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:39:14.487 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:14.487 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:14.488 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:14.488 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:39:14.488 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:14.488 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:14.489 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:14.489 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:14.489 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:39:14.489 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:14.490 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:14.490 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.859452_208.1", 2021-11-11T21:39:14.490 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:14.490 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:14.491 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:14.491 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.859452Z", 2021-11-11T21:39:14.491 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:14.491 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:14.492 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:14.492 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:14.492 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:39:14.492 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:14.493 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:14.493 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:39:14.493 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:14.493 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:14.494 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:14.494 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:39:14.494 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:14.494 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:14.494 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:14.495 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:39:14.495 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:14.495 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:39:14.495 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:39:14.496 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:39:14.496 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:14.496 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:39:14.497 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.859452_208.1 2021-11-11T21:39:19.498 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:19.499 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:19.500 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:39:19.582 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:39:19.582 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:39:20.017 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:39:20.018 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:39:20.018 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:39:20.018 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:39:20.018 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:39:20.019 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:39:20.019 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:20.019 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:20.019 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:39:20.019 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:20.020 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:39:20.020 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:20.020 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:20.020 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:39:20.021 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:20.021 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:39:20.021 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:20.021 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:39:20.021 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:20.022 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:20.022 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:20.022 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:20.022 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:39:20.023 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:20.023 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:20.023 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:39:20.023 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:20.023 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:20.024 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:20.024 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:39:20.024 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:20.024 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:20.024 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:20.025 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:20.025 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:39:20.025 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:20.025 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:20.026 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:39:20.026 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:20.026 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:20.026 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:20.027 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:39:20.027 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:20.027 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:20.027 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:20.027 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:20.028 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:39:20.028 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:20.028 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:20.028 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:39:20.029 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:20.029 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:20.029 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:20.029 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:39:20.030 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:20.030 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:20.030 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:20.031 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:39:20.031 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:20.031 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:39:20.031 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:39:20.032 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:39:20.032 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:20.032 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:39:20.033 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.825641_203.1 2021-11-11T21:39:25.035 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:25.036 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:25.037 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:39:25.085 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:39:25.085 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:39:25.355 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:39:25.355 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:39:25.355 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:39:25.356 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:39:25.356 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:39:25.356 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:39:25.356 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:25.356 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:25.357 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:39:25.357 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:25.357 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:39:25.357 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:25.358 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:25.358 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:39:25.358 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:25.358 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:39:25.358 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:25.359 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:39:25.359 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:25.359 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:25.359 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:25.360 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:25.360 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:39:25.360 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:25.360 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:25.360 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:39:25.361 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:25.361 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:25.361 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:25.361 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:39:25.361 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:25.362 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:25.362 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:25.362 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:25.362 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:39:25.363 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:25.363 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:25.363 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.050883_217.1", 2021-11-11T21:39:25.363 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:25.363 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:25.364 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:25.364 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.050883Z", 2021-11-11T21:39:25.364 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:25.364 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:25.365 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:25.365 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:25.365 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:39:25.365 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:25.365 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:25.366 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:39:25.366 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:25.366 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:25.366 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:25.367 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:39:25.367 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:25.367 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:25.368 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:25.368 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:39:25.369 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:25.369 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:39:25.369 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:39:25.369 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:39:25.369 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:25.370 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:39:25.370 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.050883_217.1 2021-11-11T21:39:30.372 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:30.372 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:30.373 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:39:30.422 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:39:30.423 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:39:30.506 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:39:30.506 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:39:30.506 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:39:30.506 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:39:30.507 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:39:30.507 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:39:30.507 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:30.507 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:30.508 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:39:30.508 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:30.508 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:39:30.508 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:30.509 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:30.509 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:39:30.509 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:30.509 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:39:30.510 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:30.510 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:39:30.510 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:30.510 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:30.510 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:30.511 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:30.511 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:39:30.511 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:30.511 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:30.512 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:39:30.512 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:30.512 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:30.512 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:30.512 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:39:30.513 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:30.513 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:30.513 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:30.513 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:30.514 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:39:30.514 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:30.514 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:30.514 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:39:30.514 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:30.515 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:30.515 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:30.515 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:39:30.515 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:30.516 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:30.516 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:30.516 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:30.516 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:39:30.516 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:30.517 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:30.517 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:39:30.517 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:30.517 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:30.518 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:30.518 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:39:30.518 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:30.518 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:30.518 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:30.519 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:39:30.519 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:30.519 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:39:30.519 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:39:30.520 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:39:30.520 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:30.520 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:39:30.521 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.825641_203.1 2021-11-11T21:39:35.522 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:35.523 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:35.524 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:39:35.580 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:39:35.581 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:39:35.668 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:39:35.668 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:39:35.668 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:39:35.668 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:39:35.669 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:39:35.669 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:39:35.669 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:35.669 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:35.670 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:39:35.670 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:35.670 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:39:35.670 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:35.671 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:35.671 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:39:35.671 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:35.671 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:39:35.672 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:35.672 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:39:35.672 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:35.672 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:35.672 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:35.673 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:35.673 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:39:35.673 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:35.673 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:35.674 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:39:35.674 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:35.674 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:35.674 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:35.674 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:39:35.675 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:35.675 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:35.675 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:35.675 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:35.676 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:39:35.676 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:35.676 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:35.677 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:39:35.677 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:35.677 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:35.677 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:35.678 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:39:35.678 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:35.678 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:35.678 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:35.678 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:35.679 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:39:35.679 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:35.679 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:35.679 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:39:35.680 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:35.680 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:35.680 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:35.680 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:39:35.680 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:35.681 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:35.681 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:35.681 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:39:35.681 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:35.682 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:39:35.682 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:39:35.682 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:39:35.682 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:35.682 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:39:35.684 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.825641_203.1 2021-11-11T21:39:40.685 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:40.686 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:40.686 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:39:40.733 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:39:40.734 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:39:40.807 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:39:40.807 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:39:40.807 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:39:40.808 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:39:40.808 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:39:40.808 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:39:40.808 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:40.808 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:40.809 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:39:40.809 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:40.809 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:39:40.809 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:40.810 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:40.810 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:39:40.810 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:40.810 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:39:40.810 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:40.811 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:39:40.811 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:40.811 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:40.811 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:40.812 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:40.812 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:39:40.812 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:40.812 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:40.812 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:39:40.813 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:40.813 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:40.813 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:40.813 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:39:40.814 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:40.814 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:40.814 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:40.814 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:40.815 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:39:40.815 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:40.815 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:40.815 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:39:40.816 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:40.816 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:40.816 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:40.816 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:39:40.816 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:40.817 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:40.817 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:40.817 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:40.817 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:39:40.817 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:40.818 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:40.818 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:39:40.818 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:40.819 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:40.819 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:40.819 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:39:40.819 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:40.820 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:40.820 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:40.820 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:39:40.820 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:40.820 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:39:40.821 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:39:40.821 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:39:40.821 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:40.821 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:39:40.822 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.789745_201.1 2021-11-11T21:39:45.824 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:45.825 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:45.825 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:39:45.872 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:39:45.872 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:39:46.155 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:39:46.155 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:39:46.155 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:39:46.156 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:39:46.156 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:39:46.156 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:39:46.156 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:46.157 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:46.157 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:39:46.157 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:46.157 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:39:46.157 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:46.158 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:46.158 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:39:46.158 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:46.158 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:39:46.159 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:46.159 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:39:46.159 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:46.160 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:46.160 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:46.160 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:46.160 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:39:46.161 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:46.161 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:46.161 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:39:46.161 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:46.162 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:46.162 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:46.162 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:39:46.162 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:46.162 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:46.163 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:46.163 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:46.163 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:39:46.163 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:46.164 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:46.164 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:39:46.164 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:46.164 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:46.165 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:46.165 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:39:46.165 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:46.165 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:46.166 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:46.166 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:46.166 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:39:46.166 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:46.167 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:46.167 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:39:46.167 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:46.167 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:46.167 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:46.168 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:39:46.168 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:46.168 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:46.168 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:46.168 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:39:46.169 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:46.169 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:39:46.169 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:39:46.169 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:39:46.170 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:46.170 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:39:46.170 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.789745_201.1 2021-11-11T21:39:51.172 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:51.173 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:51.173 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:39:51.217 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:39:51.217 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:39:51.291 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:39:51.291 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:39:51.292 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:39:51.292 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:39:51.292 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:39:51.292 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:39:51.292 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:51.293 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:51.293 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:39:51.293 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:51.293 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:39:51.294 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:51.294 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:51.294 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:39:51.294 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:51.295 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:39:51.295 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:51.295 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:39:51.295 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:51.296 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:51.296 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:51.296 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:51.296 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:39:51.297 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:51.297 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:51.297 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:39:51.297 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:51.297 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:51.298 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:51.298 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:39:51.298 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:51.298 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:51.299 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:51.299 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:51.299 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:39:51.299 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:51.299 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:51.300 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.825641_203.1", 2021-11-11T21:39:51.300 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:51.300 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:51.300 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:51.301 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.825641Z", 2021-11-11T21:39:51.301 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:51.301 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:51.301 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:51.302 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:51.302 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:39:51.302 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:51.302 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:51.303 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:39:51.303 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:51.303 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:51.303 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:51.303 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:39:51.304 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:51.304 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:51.304 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:51.304 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:39:51.304 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:51.305 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:39:51.305 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:39:51.305 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:39:51.305 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:51.306 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:39:51.307 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.825641_203.1 2021-11-11T21:39:56.308 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:56.309 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:39:56.310 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:39:56.377 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:39:56.377 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:39:56.454 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:39:56.454 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:39:56.454 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:39:56.455 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:39:56.455 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:39:56.455 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:39:56.455 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:56.456 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:56.456 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:39:56.456 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:56.456 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:39:56.457 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:56.457 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:56.457 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:39:56.457 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:56.457 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:39:56.458 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:56.458 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:39:56.458 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:56.458 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:56.459 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:56.459 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:56.459 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:39:56.459 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:56.460 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:56.460 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:39:56.460 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:56.460 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:56.461 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:56.461 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:39:56.461 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:56.461 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:56.462 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:56.462 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:56.462 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:39:56.462 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:56.462 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:56.463 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:39:56.463 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:56.463 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:56.463 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:56.464 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:39:56.464 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:56.464 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:56.464 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:56.465 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:39:56.465 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:39:56.465 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:39:56.465 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:39:56.465 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:39:56.466 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:39:56.466 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:39:56.466 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:39:56.466 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:39:56.467 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:39:56.467 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:56.467 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:56.467 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:39:56.468 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:39:56.468 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:39:56.468 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:39:56.468 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:39:56.469 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:39:56.469 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:39:56.470 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.787834_200.1 2021-11-11T21:40:01.472 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:01.473 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:01.474 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:01.531 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:01.532 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:01.825 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:01.825 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:01.825 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:01.825 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:01.826 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:01.826 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:01.826 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:01.826 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:01.827 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:01.827 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:01.827 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:01.827 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:01.828 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:01.828 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:01.828 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:01.828 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:01.828 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:01.829 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:01.829 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:01.829 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:01.829 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:01.830 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:01.830 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:01.830 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:01.830 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:01.830 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:01.831 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:01.831 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:01.831 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:01.831 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:01.832 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:01.832 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:01.832 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:01.832 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:01.832 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:01.833 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:01.833 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:01.833 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:40:01.833 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:01.834 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:01.834 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:01.834 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:40:01.834 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:01.834 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:01.835 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:01.835 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:01.835 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:01.835 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:01.836 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:01.836 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:01.836 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:01.836 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:01.836 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:01.837 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:01.837 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:01.837 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:01.837 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:01.838 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:01.838 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:01.838 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:01.838 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:01.838 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:01.839 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:01.839 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:01.839 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.768842_198.1 2021-11-11T21:40:06.840 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:06.841 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:06.842 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:06.950 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:06.950 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:07.222 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:07.222 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:07.222 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:07.223 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:07.223 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:07.223 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:07.223 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:07.224 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:07.224 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:07.224 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:07.224 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:07.224 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:07.225 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:07.225 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:07.225 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:07.225 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:07.225 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:07.226 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:07.226 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:07.226 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:07.226 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:07.226 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:07.227 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:07.227 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:07.227 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:07.227 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:07.227 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:07.228 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:07.228 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:07.228 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:07.228 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:07.228 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:07.229 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:07.229 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:07.229 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:07.229 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:07.229 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:07.230 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:40:07.230 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:07.230 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:07.230 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:07.231 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:40:07.231 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:07.231 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:07.231 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:07.231 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:07.232 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:07.232 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:07.232 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:07.232 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:07.232 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:07.233 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:07.233 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:07.233 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:07.233 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:07.234 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:07.234 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:07.234 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:07.234 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:07.234 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:07.235 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:07.235 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:07.235 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:07.235 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:07.236 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.787834_200.1 2021-11-11T21:40:12.237 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:12.238 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:12.239 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:12.439 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:12.440 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:12.955 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:12.955 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:12.955 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:12.955 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:12.956 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:12.956 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:12.956 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:12.956 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:12.957 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:12.957 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:12.957 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:12.957 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:12.957 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:12.958 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:12.958 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:12.958 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:12.958 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:12.959 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:12.959 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:12.959 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:12.959 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:12.959 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:12.960 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:12.960 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:12.960 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:12.960 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:12.960 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:12.961 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:12.961 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:12.961 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:12.961 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:12.962 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:12.962 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:12.962 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:12.962 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:12.963 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:12.963 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:12.963 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.830110_204.1", 2021-11-11T21:40:12.964 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:12.964 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:12.964 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:12.964 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.830110Z", 2021-11-11T21:40:12.965 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:12.965 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:12.965 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:12.965 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:12.965 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:12.966 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:12.966 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:12.966 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:12.966 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:12.967 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:12.968 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:12.968 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:12.968 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:12.968 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:12.969 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:12.969 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:12.970 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.830110_204.1 2021-11-11T21:40:17.972 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:17.972 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:17.973 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:18.021 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:18.021 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:18.103 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:18.103 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:18.103 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:18.104 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:18.104 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:18.104 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:18.104 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:18.105 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:18.105 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:18.105 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:18.105 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:18.106 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:18.106 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:18.106 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:18.106 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:18.107 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:18.107 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:18.107 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:18.107 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:18.107 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:18.108 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:18.108 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:18.108 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:18.108 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:18.108 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:18.109 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:18.109 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:18.109 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:18.109 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:18.109 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:18.110 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:18.110 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:18.110 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:18.110 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:18.111 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:18.111 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:18.111 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:18.111 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:40:18.111 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:18.112 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:18.112 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:18.112 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:40:18.112 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:18.112 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:18.113 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:18.113 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:18.113 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:18.114 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:18.114 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:18.114 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:18.115 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:18.115 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:18.115 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:18.115 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:18.115 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:18.116 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:18.116 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:18.116 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:18.116 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:18.116 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:18.117 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:18.117 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:18.117 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:18.117 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:18.118 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.768842_198.1 2021-11-11T21:40:23.120 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:23.121 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:23.122 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:23.164 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:23.164 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:23.235 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:23.235 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:23.235 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:23.236 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:23.236 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:23.236 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:23.236 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:23.237 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:23.237 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:23.237 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:23.237 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:23.238 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:23.238 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:23.238 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:23.238 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:23.239 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:23.239 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:23.239 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:23.239 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:23.240 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:23.240 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:23.240 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:23.240 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:23.240 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:23.241 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:23.241 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:23.241 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:23.241 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:23.242 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:23.242 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:23.242 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:23.242 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:23.243 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:23.243 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:23.243 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:23.243 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:23.243 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:23.244 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.977739_213.1", 2021-11-11T21:40:23.244 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:23.244 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:23.244 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:23.245 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.977739Z", 2021-11-11T21:40:23.245 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:23.245 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:23.245 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:23.246 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:23.246 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:23.246 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:23.246 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:23.247 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:23.247 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:23.247 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:23.247 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:23.248 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:23.248 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:23.248 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:23.248 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:23.248 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:23.249 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:23.249 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:23.249 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:23.249 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:23.250 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:23.250 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:23.251 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.977739_213.1 2021-11-11T21:40:28.253 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:28.254 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:28.254 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:28.352 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:28.352 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:28.645 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:28.646 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:28.646 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:28.646 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:28.646 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:28.647 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:28.647 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:28.647 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:28.647 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:28.648 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:28.648 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:28.648 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:28.648 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:28.649 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:28.649 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:28.649 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:28.649 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:28.650 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:28.650 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:28.650 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:28.650 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:28.650 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:28.651 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:28.651 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:28.651 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:28.651 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:28.652 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:28.652 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:28.652 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:28.652 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:28.653 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:28.653 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:28.653 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:28.653 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:28.653 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:28.654 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:28.654 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:28.654 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:40:28.654 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:28.655 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:28.655 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:28.655 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:40:28.655 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:28.656 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:28.656 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:28.656 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:28.656 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:28.656 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:28.657 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:28.657 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:28.657 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:28.657 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:28.658 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:28.658 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:28.658 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:28.658 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:28.659 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:28.659 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:28.659 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:28.659 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:28.660 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:28.660 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:28.660 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:28.660 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:28.661 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.789745_201.1 2021-11-11T21:40:33.662 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:33.663 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:33.664 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:33.742 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:33.743 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:33.881 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:33.881 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:33.881 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:33.882 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:33.882 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:33.882 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:33.882 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:33.883 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:33.883 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:33.883 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:33.883 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:33.883 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:33.884 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:33.884 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:33.884 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:33.884 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:33.884 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:33.885 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:33.885 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:33.885 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:33.885 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:33.886 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:33.886 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:33.886 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:33.886 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:33.886 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:33.887 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:33.887 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:33.887 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:33.887 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:33.888 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:33.888 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:33.888 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:33.888 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:33.888 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:33.889 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:33.889 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:33.889 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:40:33.889 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:33.889 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:33.890 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:33.890 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:40:33.890 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:33.890 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:33.890 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:33.891 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:33.891 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:33.891 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:33.891 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:33.892 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:33.892 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:33.892 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:33.892 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:33.892 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:33.893 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:33.893 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:33.893 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:33.893 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:33.893 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:33.894 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:33.894 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:33.894 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:33.894 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:33.895 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:33.896 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:40:38.897 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:38.898 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:38.898 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:38.946 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:38.946 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:39.030 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:39.031 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:39.031 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:39.031 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:39.031 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:39.032 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:39.032 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:39.032 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:39.032 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:39.033 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:39.033 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:39.033 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:39.033 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:39.033 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:39.034 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:39.034 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:39.034 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:39.034 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:39.035 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:39.035 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:39.035 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:39.035 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:39.035 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:39.036 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:39.036 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:39.036 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:39.036 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:39.037 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:39.037 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:39.037 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:39.037 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:39.037 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:39.038 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:39.038 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:39.038 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:39.038 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:39.039 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:39.039 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665225.075927_221.1", 2021-11-11T21:40:39.039 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:39.039 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:39.039 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:39.040 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:45.075927Z", 2021-11-11T21:40:39.040 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:39.040 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:39.040 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:39.041 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:39.041 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:39.041 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:39.041 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:39.041 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:39.042 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:39.042 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:39.042 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:39.042 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:39.042 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:39.043 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:39.043 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:39.043 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:39.043 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:39.043 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:39.044 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:39.044 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:39.044 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:39.044 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:39.045 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665225.075927_221.1 2021-11-11T21:40:44.046 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:44.047 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:44.048 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:44.092 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:44.093 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:44.164 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:44.164 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:44.165 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:44.165 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:44.165 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:44.165 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:44.166 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:44.166 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:44.166 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:44.166 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:44.167 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:44.167 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:44.167 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:44.167 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:44.168 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:44.168 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:44.168 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:44.168 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:44.169 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:44.169 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:44.169 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:44.169 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:44.170 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:44.170 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:44.170 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:44.170 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:44.170 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:44.171 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:44.171 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:44.171 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:44.171 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:44.172 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:44.172 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:44.172 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:44.172 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:44.173 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:44.173 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:44.173 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.787834_200.1", 2021-11-11T21:40:44.173 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:44.174 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:44.174 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:44.174 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.787834Z", 2021-11-11T21:40:44.174 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:44.174 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:44.175 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:44.175 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:44.175 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:44.175 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:44.176 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:44.176 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:44.176 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:44.176 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:44.177 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:44.177 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:44.177 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:44.177 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:44.178 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:44.178 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:44.178 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:44.178 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:44.179 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:44.179 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:44.179 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:44.179 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:44.180 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.787834_200.1 2021-11-11T21:40:49.182 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:49.183 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:49.183 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:49.263 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:49.263 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:49.386 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:49.387 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:49.387 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:49.387 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:49.388 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:49.388 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:49.388 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:49.388 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:49.389 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:49.389 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:49.389 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:49.389 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:49.389 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:49.390 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:49.390 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:49.390 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:49.390 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:49.391 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:49.391 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:49.391 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:49.391 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:49.392 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:49.392 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:49.392 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:49.392 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:49.392 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:49.393 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:49.393 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:49.393 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:49.393 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:49.394 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:49.394 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:49.394 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:49.395 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:49.395 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:49.395 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:49.395 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:49.395 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.859452_208.1", 2021-11-11T21:40:49.396 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:49.396 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:49.396 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:49.396 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.859452Z", 2021-11-11T21:40:49.397 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:49.397 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:49.397 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:49.397 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:49.397 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:49.398 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:49.398 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:49.398 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:49.398 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:49.399 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:49.399 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:49.399 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:49.399 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:49.400 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:49.400 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:49.400 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:49.400 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:49.401 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:49.401 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:49.401 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:49.401 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:49.401 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:49.403 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.859452_208.1 2021-11-11T21:40:54.404 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:54.405 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:54.406 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:54.507 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:54.507 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:54.624 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:54.684 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:54.684 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:54.685 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:54.685 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:54.685 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:54.685 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:54.686 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:54.686 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:54.686 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:54.686 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:54.687 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:54.687 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:54.687 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:54.687 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:54.688 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:54.688 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:54.688 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:54.688 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:54.689 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:54.689 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:54.689 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:54.689 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:54.689 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:54.690 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:54.690 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:54.690 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:54.690 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:54.691 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:54.691 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:54.691 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:54.691 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:54.692 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:54.692 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:54.692 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:54.693 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:54.693 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:54.693 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:40:54.693 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:54.693 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:54.694 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:54.694 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:40:54.694 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:54.694 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:54.695 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:54.695 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:54.695 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:54.695 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:54.695 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:54.696 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:54.696 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:54.696 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:54.696 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:54.697 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:54.697 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:54.697 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:54.697 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:54.698 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:54.698 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:54.698 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:54.698 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:54.698 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:54.699 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:54.699 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:54.700 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.831934_205.1 2021-11-11T21:40:59.701 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:59.702 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:40:59.703 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:40:59.749 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:40:59.750 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:40:59.828 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:40:59.829 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:40:59.829 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:40:59.829 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:40:59.829 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:40:59.830 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:40:59.830 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:59.830 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:59.830 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:40:59.831 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:59.831 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:40:59.831 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:59.831 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:59.832 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:40:59.832 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:59.832 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:40:59.832 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:59.833 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:40:59.833 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:59.833 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:59.833 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:59.833 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:59.834 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:40:59.834 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:59.834 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:59.834 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:40:59.835 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:59.835 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:59.835 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:59.835 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:40:59.836 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:59.836 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:59.836 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:59.836 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:59.836 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:40:59.837 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:59.837 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:59.837 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.917826_211.1", 2021-11-11T21:40:59.837 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:59.838 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:59.838 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:59.838 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.917826Z", 2021-11-11T21:40:59.838 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:59.838 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:59.839 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:59.839 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:40:59.839 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:40:59.839 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:40:59.840 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:40:59.840 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:40:59.840 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:40:59.840 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:40:59.841 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:40:59.841 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:40:59.841 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:40:59.841 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:59.842 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:59.842 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:40:59.842 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:40:59.842 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:40:59.843 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:40:59.843 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:40:59.843 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:40:59.843 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:40:59.845 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.917826_211.1 2021-11-11T21:41:04.846 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:41:04.846 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:41:04.847 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:41:04.910 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:41:04.910 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:41:05.076 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:41:05.077 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:41:05.077 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:41:05.077 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:41:05.077 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:41:05.078 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:41:05.078 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:05.078 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:05.078 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:41:05.079 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:05.079 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:41:05.079 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:05.079 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:05.080 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:41:05.080 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:05.080 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:41:05.080 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:05.080 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:41:05.081 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:05.081 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:05.081 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:05.081 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:05.082 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:41:05.082 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:05.082 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:05.082 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:41:05.082 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:05.083 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:05.083 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:05.083 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:41:05.083 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:05.084 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:05.084 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:05.084 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:05.084 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:41:05.085 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:05.085 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:05.085 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.831934_205.1", 2021-11-11T21:41:05.085 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:05.085 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:05.086 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:05.086 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.831934Z", 2021-11-11T21:41:05.086 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:05.086 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:05.086 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:05.087 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:05.087 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:41:05.087 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:05.087 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:05.088 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:41:05.088 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:05.088 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:05.088 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:05.088 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:41:05.089 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:05.089 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:05.089 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:05.089 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:41:05.089 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:05.090 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:41:05.090 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:41:05.090 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:41:05.090 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:05.091 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:41:05.092 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.831934_205.1 2021-11-11T21:41:10.093 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:41:10.094 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:41:10.094 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:41:10.156 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:41:10.157 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:41:10.627 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:41:10.628 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:41:10.628 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:41:10.628 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:41:10.628 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:41:10.628 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:41:10.629 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:10.629 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:10.629 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:41:10.629 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:10.630 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:41:10.630 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:10.630 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:10.630 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:41:10.631 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:10.631 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:41:10.631 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:10.631 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:41:10.631 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:10.632 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:10.632 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:10.632 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:10.632 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:41:10.633 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:10.633 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:10.633 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:41:10.633 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:10.633 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:10.634 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:10.634 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:41:10.634 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:10.634 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:10.635 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:10.635 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:10.635 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:41:10.635 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:10.635 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:10.636 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:41:10.636 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:10.636 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:10.636 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:10.636 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:41:10.637 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:10.637 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:10.637 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:10.637 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:10.638 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:41:10.638 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:10.638 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:10.638 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:41:10.639 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:10.639 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:10.639 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:10.639 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:41:10.640 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:10.640 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:10.640 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:10.640 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:41:10.641 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:10.641 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:41:10.641 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:41:10.641 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:41:10.641 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:10.642 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:41:10.643 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.789745_201.1 2021-11-11T21:41:15.645 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:41:15.645 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:41:15.646 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:41:15.709 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:41:15.709 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:41:15.800 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:41:15.800 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:41:15.800 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:41:15.801 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:41:15.801 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:41:15.801 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:41:15.801 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:15.802 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:15.802 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:41:15.802 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:15.802 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:41:15.803 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:15.803 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:15.803 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:41:15.803 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:15.803 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:41:15.804 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:15.804 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:41:15.804 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:15.804 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:15.805 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:15.805 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:15.805 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:41:15.805 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:15.805 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:15.806 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:41:15.806 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:15.806 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:15.806 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:15.807 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:41:15.807 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:15.807 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:15.807 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:15.807 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:15.808 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:41:15.808 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:15.808 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:15.808 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.768842_198.1", 2021-11-11T21:41:15.809 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:15.809 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:15.809 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:15.809 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.768842Z", 2021-11-11T21:41:15.809 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:15.810 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:15.810 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:15.810 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:15.810 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:41:15.811 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:15.811 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:15.811 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:41:15.811 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:15.811 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:15.812 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:15.812 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:41:15.812 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:15.812 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:15.813 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:15.813 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:41:15.813 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:15.813 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:41:15.813 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:41:15.814 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:41:15.814 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:15.814 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:41:15.815 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.768842_198.1 2021-11-11T21:41:20.817 INFO:tasks.util.rgw:rgwadmin: c2.client.0 : ['metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:41:20.817 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c2', 'metadata', 'sync', 'status', '--rgw-zone', 'test-zone2', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c2', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T21:41:20.818 DEBUG:teuthology.orchestra.run.smithi198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c2 metadata sync status --rgw-zone test-zone2 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c2 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T21:41:20.914 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:41:20.915 INFO:teuthology.orchestra.run.smithi198.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:41:21.365 INFO:teuthology.orchestra.run.smithi198.stdout:{ 2021-11-11T21:41:21.365 INFO:teuthology.orchestra.run.smithi198.stdout: "sync_status": { 2021-11-11T21:41:21.366 INFO:teuthology.orchestra.run.smithi198.stdout: "info": { 2021-11-11T21:41:21.366 INFO:teuthology.orchestra.run.smithi198.stdout: "status": "sync", 2021-11-11T21:41:21.366 INFO:teuthology.orchestra.run.smithi198.stdout: "num_shards": 4, 2021-11-11T21:41:21.366 INFO:teuthology.orchestra.run.smithi198.stdout: "period": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:41:21.366 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:21.367 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:21.367 INFO:teuthology.orchestra.run.smithi198.stdout: "markers": [ 2021-11-11T21:41:21.367 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:21.367 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 0, 2021-11-11T21:41:21.368 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:21.368 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:21.368 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636664980.616234_46.1", 2021-11-11T21:41:21.368 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:21.368 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 1, 2021-11-11T21:41:21.369 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:21.369 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:09:40.616234Z", 2021-11-11T21:41:21.369 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:21.369 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:21.370 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:21.370 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:21.370 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 1, 2021-11-11T21:41:21.370 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:21.370 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:21.371 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "", 2021-11-11T21:41:21.371 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:21.371 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:21.371 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:21.372 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "0.000000", 2021-11-11T21:41:21.372 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:21.372 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:21.372 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:21.372 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:21.373 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 2, 2021-11-11T21:41:21.373 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:21.373 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:21.373 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665224.789745_201.1", 2021-11-11T21:41:21.373 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:21.374 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:21.374 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:21.374 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:13:44.789745Z", 2021-11-11T21:41:21.374 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:21.375 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:21.375 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:21.375 INFO:teuthology.orchestra.run.smithi198.stdout: { 2021-11-11T21:41:21.375 INFO:teuthology.orchestra.run.smithi198.stdout: "key": 3, 2021-11-11T21:41:21.375 INFO:teuthology.orchestra.run.smithi198.stdout: "val": { 2021-11-11T21:41:21.376 INFO:teuthology.orchestra.run.smithi198.stdout: "state": 1, 2021-11-11T21:41:21.376 INFO:teuthology.orchestra.run.smithi198.stdout: "marker": "1_1636665018.067753_86.1", 2021-11-11T21:41:21.376 INFO:teuthology.orchestra.run.smithi198.stdout: "next_step_marker": "", 2021-11-11T21:41:21.376 INFO:teuthology.orchestra.run.smithi198.stdout: "total_entries": 0, 2021-11-11T21:41:21.377 INFO:teuthology.orchestra.run.smithi198.stdout: "pos": 0, 2021-11-11T21:41:21.377 INFO:teuthology.orchestra.run.smithi198.stdout: "timestamp": "2021-11-11T21:10:18.067753Z", 2021-11-11T21:41:21.377 INFO:teuthology.orchestra.run.smithi198.stdout: "realm_epoch": 2 2021-11-11T21:41:21.377 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:21.377 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:21.378 INFO:teuthology.orchestra.run.smithi198.stdout: ] 2021-11-11T21:41:21.378 INFO:teuthology.orchestra.run.smithi198.stdout: }, 2021-11-11T21:41:21.378 INFO:teuthology.orchestra.run.smithi198.stdout: "full_sync": { 2021-11-11T21:41:21.378 INFO:teuthology.orchestra.run.smithi198.stdout: "total": 1, 2021-11-11T21:41:21.379 INFO:teuthology.orchestra.run.smithi198.stdout: "complete": 1 2021-11-11T21:41:21.379 INFO:teuthology.orchestra.run.smithi198.stdout: } 2021-11-11T21:41:21.379 INFO:teuthology.orchestra.run.smithi198.stdout:} 2021-11-11T21:41:21.380 WARNING:rgw_multi.tests:zone test-zone2 behind master: shard=2 master=1_1636666253.051934_329.1 target=1_1636665224.789745_201.1 2021-11-11T21:41:26.384 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_zonegroup_remove ... FAIL 2021-11-11T21:41:26.385 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'create', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:26.386 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'create', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:26.387 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup create --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:41:26.676 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:41:26.676 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:41:26.714 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:41:26.715 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "fa34a3d5-e9e3-4d20-8699-4614401328cc", 2021-11-11T21:41:26.715 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "remove_zg", 2021-11-11T21:41:26.715 INFO:teuthology.orchestra.run.smithi096.stdout: "api_name": "remove_zg", 2021-11-11T21:41:26.715 INFO:teuthology.orchestra.run.smithi096.stdout: "is_master": "false", 2021-11-11T21:41:26.715 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [], 2021-11-11T21:41:26.716 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames": [], 2021-11-11T21:41:26.716 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames_s3website": [], 2021-11-11T21:41:26.716 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zone": "", 2021-11-11T21:41:26.716 INFO:teuthology.orchestra.run.smithi096.stdout: "zones": [], 2021-11-11T21:41:26.717 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_targets": [], 2021-11-11T21:41:26.717 INFO:teuthology.orchestra.run.smithi096.stdout: "default_placement": "", 2021-11-11T21:41:26.717 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:41:26.717 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_policy": { 2021-11-11T21:41:26.717 INFO:teuthology.orchestra.run.smithi096.stdout: "groups": [] 2021-11-11T21:41:26.718 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:26.718 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:41:26.719 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'create', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:26.719 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'create', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:26.720 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone create --rgw-zone remove --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:41:26.761 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:41:26.761 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:41:26.800 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:41:26.800 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "b2e5cfd8-cfbb-46f8-8458-326b29dd6700", 2021-11-11T21:41:26.800 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "remove", 2021-11-11T21:41:26.800 INFO:teuthology.orchestra.run.smithi096.stdout: "domain_root": "remove.rgw.meta:root", 2021-11-11T21:41:26.801 INFO:teuthology.orchestra.run.smithi096.stdout: "control_pool": "remove.rgw.control", 2021-11-11T21:41:26.801 INFO:teuthology.orchestra.run.smithi096.stdout: "gc_pool": "remove.rgw.log:gc", 2021-11-11T21:41:26.801 INFO:teuthology.orchestra.run.smithi096.stdout: "lc_pool": "remove.rgw.log:lc", 2021-11-11T21:41:26.801 INFO:teuthology.orchestra.run.smithi096.stdout: "log_pool": "remove.rgw.log", 2021-11-11T21:41:26.802 INFO:teuthology.orchestra.run.smithi096.stdout: "intent_log_pool": "remove.rgw.log:intent", 2021-11-11T21:41:26.802 INFO:teuthology.orchestra.run.smithi096.stdout: "usage_log_pool": "remove.rgw.log:usage", 2021-11-11T21:41:26.802 INFO:teuthology.orchestra.run.smithi096.stdout: "roles_pool": "remove.rgw.meta:roles", 2021-11-11T21:41:26.802 INFO:teuthology.orchestra.run.smithi096.stdout: "reshard_pool": "remove.rgw.log:reshard", 2021-11-11T21:41:26.802 INFO:teuthology.orchestra.run.smithi096.stdout: "user_keys_pool": "remove.rgw.meta:users.keys", 2021-11-11T21:41:26.803 INFO:teuthology.orchestra.run.smithi096.stdout: "user_email_pool": "remove.rgw.meta:users.email", 2021-11-11T21:41:26.803 INFO:teuthology.orchestra.run.smithi096.stdout: "user_swift_pool": "remove.rgw.meta:users.swift", 2021-11-11T21:41:26.803 INFO:teuthology.orchestra.run.smithi096.stdout: "user_uid_pool": "remove.rgw.meta:users.uid", 2021-11-11T21:41:26.803 INFO:teuthology.orchestra.run.smithi096.stdout: "otp_pool": "remove.rgw.otp", 2021-11-11T21:41:26.804 INFO:teuthology.orchestra.run.smithi096.stdout: "system_key": { 2021-11-11T21:41:26.804 INFO:teuthology.orchestra.run.smithi096.stdout: "access_key": "", 2021-11-11T21:41:26.804 INFO:teuthology.orchestra.run.smithi096.stdout: "secret_key": "" 2021-11-11T21:41:26.804 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:26.804 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_pools": [ 2021-11-11T21:41:26.805 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:26.805 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "default-placement", 2021-11-11T21:41:26.805 INFO:teuthology.orchestra.run.smithi096.stdout: "val": { 2021-11-11T21:41:26.805 INFO:teuthology.orchestra.run.smithi096.stdout: "index_pool": "remove.rgw.buckets.index", 2021-11-11T21:41:26.805 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": { 2021-11-11T21:41:26.806 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD": { 2021-11-11T21:41:26.806 INFO:teuthology.orchestra.run.smithi096.stdout: "data_pool": "remove.rgw.buckets.data" 2021-11-11T21:41:26.806 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:26.806 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:26.807 INFO:teuthology.orchestra.run.smithi096.stdout: "data_extra_pool": "remove.rgw.buckets.non-ec", 2021-11-11T21:41:26.807 INFO:teuthology.orchestra.run.smithi096.stdout: "index_type": 0 2021-11-11T21:41:26.807 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:26.807 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:26.807 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:26.808 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:41:26.808 INFO:teuthology.orchestra.run.smithi096.stdout: "notif_pool": "remove.rgw.log:notif" 2021-11-11T21:41:26.808 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:41:26.809 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:26.810 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:26.810 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:41:26.891 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:41:26.892 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:41:26.994 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:41:26.994 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:41:26.995 INFO:teuthology.orchestra.run.smithi096.stdout: "epoch": 6, 2021-11-11T21:41:26.995 INFO:teuthology.orchestra.run.smithi096.stdout: "predecessor_uuid": "ab45263b-38f8-4e1a-9f37-1e83128a94ad", 2021-11-11T21:41:26.995 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": [], 2021-11-11T21:41:26.995 INFO:teuthology.orchestra.run.smithi096.stdout: "period_map": { 2021-11-11T21:41:26.996 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:41:26.996 INFO:teuthology.orchestra.run.smithi096.stdout: "zonegroups": [ 2021-11-11T21:41:26.996 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:26.996 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "fa34a3d5-e9e3-4d20-8699-4614401328cc", 2021-11-11T21:41:26.997 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "remove_zg", 2021-11-11T21:41:26.997 INFO:teuthology.orchestra.run.smithi096.stdout: "api_name": "remove_zg", 2021-11-11T21:41:26.997 INFO:teuthology.orchestra.run.smithi096.stdout: "is_master": "false", 2021-11-11T21:41:26.997 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [], 2021-11-11T21:41:26.998 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames": [], 2021-11-11T21:41:26.998 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames_s3website": [], 2021-11-11T21:41:26.998 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zone": "b2e5cfd8-cfbb-46f8-8458-326b29dd6700", 2021-11-11T21:41:26.998 INFO:teuthology.orchestra.run.smithi096.stdout: "zones": [ 2021-11-11T21:41:26.999 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:26.999 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "b2e5cfd8-cfbb-46f8-8458-326b29dd6700", 2021-11-11T21:41:26.999 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "remove", 2021-11-11T21:41:26.999 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [], 2021-11-11T21:41:27.000 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:41:27.000 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "false", 2021-11-11T21:41:27.000 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:41:27.000 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:41:27.001 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "", 2021-11-11T21:41:27.001 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "true", 2021-11-11T21:41:27.001 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [], 2021-11-11T21:41:27.001 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:41:27.002 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.002 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.002 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_targets": [ 2021-11-11T21:41:27.003 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.003 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "default-placement", 2021-11-11T21:41:27.003 INFO:teuthology.orchestra.run.smithi096.stdout: "tags": [], 2021-11-11T21:41:27.003 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": [ 2021-11-11T21:41:27.003 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD" 2021-11-11T21:41:27.004 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:41:27.004 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.004 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.004 INFO:teuthology.orchestra.run.smithi096.stdout: "default_placement": "default-placement", 2021-11-11T21:41:27.005 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:41:27.005 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_policy": { 2021-11-11T21:41:27.005 INFO:teuthology.orchestra.run.smithi096.stdout: "groups": [] 2021-11-11T21:41:27.005 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.005 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.006 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.006 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "test-zonegroup", 2021-11-11T21:41:27.006 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zonegroup", 2021-11-11T21:41:27.007 INFO:teuthology.orchestra.run.smithi096.stdout: "api_name": "", 2021-11-11T21:41:27.007 INFO:teuthology.orchestra.run.smithi096.stdout: "is_master": "true", 2021-11-11T21:41:27.007 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:41:27.007 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:41:27.008 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.008 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames": [], 2021-11-11T21:41:27.008 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames_s3website": [], 2021-11-11T21:41:27.008 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:41:27.009 INFO:teuthology.orchestra.run.smithi096.stdout: "zones": [ 2021-11-11T21:41:27.009 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.009 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "31f4b03b-e9bc-4500-a988-e29031fe0663", 2021-11-11T21:41:27.009 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone4", 2021-11-11T21:41:27.009 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:41:27.010 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi198.front.sepia.ceph.com:8001" 2021-11-11T21:41:27.010 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.010 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:41:27.010 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "true", 2021-11-11T21:41:27.011 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:41:27.011 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:41:27.012 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "pubsub", 2021-11-11T21:41:27.012 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "false", 2021-11-11T21:41:27.013 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [ 2021-11-11T21:41:27.013 INFO:teuthology.orchestra.run.smithi096.stdout: "test-zone1" 2021-11-11T21:41:27.013 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.014 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:41:27.014 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.014 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.015 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:41:27.015 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone3", 2021-11-11T21:41:27.015 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:41:27.016 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8001" 2021-11-11T21:41:27.016 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.016 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:41:27.017 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "true", 2021-11-11T21:41:27.017 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:41:27.017 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:41:27.017 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "", 2021-11-11T21:41:27.018 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "true", 2021-11-11T21:41:27.018 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [], 2021-11-11T21:41:27.018 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:41:27.018 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.019 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.019 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:41:27.019 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone2", 2021-11-11T21:41:27.019 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:41:27.020 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi198.front.sepia.ceph.com:8000" 2021-11-11T21:41:27.020 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.021 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:41:27.021 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "true", 2021-11-11T21:41:27.021 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:41:27.021 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:41:27.024 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "", 2021-11-11T21:41:27.025 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "false", 2021-11-11T21:41:27.025 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [], 2021-11-11T21:41:27.025 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:41:27.025 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.026 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.026 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:41:27.026 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone1", 2021-11-11T21:41:27.026 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:41:27.027 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:41:27.027 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.027 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:41:27.027 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "true", 2021-11-11T21:41:27.028 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:41:27.028 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:41:27.028 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "", 2021-11-11T21:41:27.028 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "true", 2021-11-11T21:41:27.028 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [], 2021-11-11T21:41:27.029 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:41:27.029 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.029 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.029 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_targets": [ 2021-11-11T21:41:27.030 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.030 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "default-placement", 2021-11-11T21:41:27.030 INFO:teuthology.orchestra.run.smithi096.stdout: "tags": [], 2021-11-11T21:41:27.030 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": [ 2021-11-11T21:41:27.031 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD" 2021-11-11T21:41:27.031 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:41:27.031 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.031 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.032 INFO:teuthology.orchestra.run.smithi096.stdout: "default_placement": "default-placement", 2021-11-11T21:41:27.032 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:41:27.032 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_policy": { 2021-11-11T21:41:27.032 INFO:teuthology.orchestra.run.smithi096.stdout: "groups": [] 2021-11-11T21:41:27.032 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.033 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.033 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.033 INFO:teuthology.orchestra.run.smithi096.stdout: "short_zone_ids": [ 2021-11-11T21:41:27.034 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.034 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "31f4b03b-e9bc-4500-a988-e29031fe0663", 2021-11-11T21:41:27.035 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 2936728019 2021-11-11T21:41:27.035 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.035 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.036 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:41:27.036 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 986920242 2021-11-11T21:41:27.036 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.036 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.037 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:41:27.037 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 1380659451 2021-11-11T21:41:27.037 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.037 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.038 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "b2e5cfd8-cfbb-46f8-8458-326b29dd6700", 2021-11-11T21:41:27.038 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 3396947722 2021-11-11T21:41:27.038 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.038 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.038 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:41:27.039 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 1844113099 2021-11-11T21:41:27.039 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.039 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:41:27.039 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.040 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zonegroup": "test-zonegroup", 2021-11-11T21:41:27.040 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:41:27.040 INFO:teuthology.orchestra.run.smithi096.stdout: "period_config": { 2021-11-11T21:41:27.040 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_quota": { 2021-11-11T21:41:27.041 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T21:41:27.041 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T21:41:27.041 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T21:41:27.041 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T21:41:27.041 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T21:41:27.042 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.042 INFO:teuthology.orchestra.run.smithi096.stdout: "user_quota": { 2021-11-11T21:41:27.042 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T21:41:27.042 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T21:41:27.043 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T21:41:27.043 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T21:41:27.043 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T21:41:27.043 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.044 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.044 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:41:27.044 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_name": "test-realm", 2021-11-11T21:41:27.044 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:41:27.044 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:41:27.046 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zone', 'delete', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:27.047 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zone', 'delete', '--rgw-zone', 'remove', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:27.048 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zone delete --rgw-zone remove --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:41:27.130 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:41:27.130 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:41:27.182 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:41:27.181+0000 7f831d1a7380 0 zone id b2e5cfd8-cfbb-46f8-8458-326b29dd6700 is not a part of zonegroup test-zonegroup 2021-11-11T21:41:27.183 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:41:27.182+0000 7f831d1a7380 0 zone id b2e5cfd8-cfbb-46f8-8458-326b29dd6700 is not a part of zonegroup default 2021-11-11T21:41:27.191 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:27.192 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:27.192 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:41:27.240 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:41:27.240 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:41:27.288 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T21:41:27.287+0000 7f7adb0b7380 0 ERROR: zonegroup remove_zg has a non existent master zone 2021-11-11T21:41:27.292 DEBUG:teuthology.orchestra.run:got remote process result: 22 2021-11-11T21:41:27.293 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['zonegroup', 'delete', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:27.293 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'zonegroup', 'delete', '--rgw-zonegroup', 'remove_zg', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:27.294 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 zonegroup delete --rgw-zonegroup remove_zg --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:41:27.354 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:41:27.354 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:41:27.434 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:27.435 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'period', 'update', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--commit', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0'] 2021-11-11T21:41:27.435 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 period update --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --commit --cluster c1 --debug-rgw 0 --debug-ms 0 2021-11-11T21:41:27.499 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T21:41:27.500 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T21:41:27.715 INFO:teuthology.orchestra.run.smithi096.stdout:{ 2021-11-11T21:41:27.716 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:41:27.716 INFO:teuthology.orchestra.run.smithi096.stdout: "epoch": 7, 2021-11-11T21:41:27.716 INFO:teuthology.orchestra.run.smithi096.stdout: "predecessor_uuid": "ab45263b-38f8-4e1a-9f37-1e83128a94ad", 2021-11-11T21:41:27.716 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_status": [], 2021-11-11T21:41:27.716 INFO:teuthology.orchestra.run.smithi096.stdout: "period_map": { 2021-11-11T21:41:27.717 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "2aa9d15b-21de-40bc-a7d3-3d5e5933111b", 2021-11-11T21:41:27.717 INFO:teuthology.orchestra.run.smithi096.stdout: "zonegroups": [ 2021-11-11T21:41:27.717 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.717 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "test-zonegroup", 2021-11-11T21:41:27.718 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zonegroup", 2021-11-11T21:41:27.718 INFO:teuthology.orchestra.run.smithi096.stdout: "api_name": "", 2021-11-11T21:41:27.718 INFO:teuthology.orchestra.run.smithi096.stdout: "is_master": "true", 2021-11-11T21:41:27.718 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:41:27.718 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:41:27.719 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.719 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames": [], 2021-11-11T21:41:27.719 INFO:teuthology.orchestra.run.smithi096.stdout: "hostnames_s3website": [], 2021-11-11T21:41:27.719 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:41:27.720 INFO:teuthology.orchestra.run.smithi096.stdout: "zones": [ 2021-11-11T21:41:27.720 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.720 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "31f4b03b-e9bc-4500-a988-e29031fe0663", 2021-11-11T21:41:27.720 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone4", 2021-11-11T21:41:27.720 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:41:27.721 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi198.front.sepia.ceph.com:8001" 2021-11-11T21:41:27.721 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.721 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:41:27.721 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "true", 2021-11-11T21:41:27.722 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:41:27.722 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:41:27.722 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "pubsub", 2021-11-11T21:41:27.722 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "false", 2021-11-11T21:41:27.722 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [ 2021-11-11T21:41:27.723 INFO:teuthology.orchestra.run.smithi096.stdout: "test-zone1" 2021-11-11T21:41:27.723 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.723 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:41:27.723 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.723 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.724 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:41:27.724 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone3", 2021-11-11T21:41:27.724 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:41:27.724 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8001" 2021-11-11T21:41:27.725 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.725 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:41:27.725 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "true", 2021-11-11T21:41:27.725 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:41:27.725 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:41:27.726 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "", 2021-11-11T21:41:27.726 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "true", 2021-11-11T21:41:27.726 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [], 2021-11-11T21:41:27.726 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:41:27.727 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.727 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.727 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:41:27.727 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone2", 2021-11-11T21:41:27.727 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:41:27.728 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi198.front.sepia.ceph.com:8000" 2021-11-11T21:41:27.728 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.728 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:41:27.728 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "true", 2021-11-11T21:41:27.729 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:41:27.729 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:41:27.729 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "", 2021-11-11T21:41:27.729 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "false", 2021-11-11T21:41:27.729 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [], 2021-11-11T21:41:27.730 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:41:27.730 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.730 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.730 INFO:teuthology.orchestra.run.smithi096.stdout: "id": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:41:27.731 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "test-zone1", 2021-11-11T21:41:27.731 INFO:teuthology.orchestra.run.smithi096.stdout: "endpoints": [ 2021-11-11T21:41:27.731 INFO:teuthology.orchestra.run.smithi096.stdout: "http://smithi096.front.sepia.ceph.com:8000" 2021-11-11T21:41:27.731 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.731 INFO:teuthology.orchestra.run.smithi096.stdout: "log_meta": "false", 2021-11-11T21:41:27.732 INFO:teuthology.orchestra.run.smithi096.stdout: "log_data": "true", 2021-11-11T21:41:27.732 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_index_max_shards": 11, 2021-11-11T21:41:27.732 INFO:teuthology.orchestra.run.smithi096.stdout: "read_only": "false", 2021-11-11T21:41:27.732 INFO:teuthology.orchestra.run.smithi096.stdout: "tier_type": "", 2021-11-11T21:41:27.732 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from_all": "true", 2021-11-11T21:41:27.733 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_from": [], 2021-11-11T21:41:27.733 INFO:teuthology.orchestra.run.smithi096.stdout: "redirect_zone": "" 2021-11-11T21:41:27.733 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.733 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.733 INFO:teuthology.orchestra.run.smithi096.stdout: "placement_targets": [ 2021-11-11T21:41:27.734 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.734 INFO:teuthology.orchestra.run.smithi096.stdout: "name": "default-placement", 2021-11-11T21:41:27.734 INFO:teuthology.orchestra.run.smithi096.stdout: "tags": [], 2021-11-11T21:41:27.734 INFO:teuthology.orchestra.run.smithi096.stdout: "storage_classes": [ 2021-11-11T21:41:27.735 INFO:teuthology.orchestra.run.smithi096.stdout: "STANDARD" 2021-11-11T21:41:27.735 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:41:27.735 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.735 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.735 INFO:teuthology.orchestra.run.smithi096.stdout: "default_placement": "default-placement", 2021-11-11T21:41:27.736 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:41:27.736 INFO:teuthology.orchestra.run.smithi096.stdout: "sync_policy": { 2021-11-11T21:41:27.736 INFO:teuthology.orchestra.run.smithi096.stdout: "groups": [] 2021-11-11T21:41:27.736 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.736 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.737 INFO:teuthology.orchestra.run.smithi096.stdout: ], 2021-11-11T21:41:27.737 INFO:teuthology.orchestra.run.smithi096.stdout: "short_zone_ids": [ 2021-11-11T21:41:27.737 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.757 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "31f4b03b-e9bc-4500-a988-e29031fe0663", 2021-11-11T21:41:27.758 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 2936728019 2021-11-11T21:41:27.758 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.758 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.758 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "4a40cc50-357f-4f11-a692-5925070095e8", 2021-11-11T21:41:27.759 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 986920242 2021-11-11T21:41:27.759 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.759 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.759 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "82b05a71-8273-4ec7-b7c6-953c0f40d3e0", 2021-11-11T21:41:27.760 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 1380659451 2021-11-11T21:41:27.760 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.760 INFO:teuthology.orchestra.run.smithi096.stdout: { 2021-11-11T21:41:27.760 INFO:teuthology.orchestra.run.smithi096.stdout: "key": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:41:27.760 INFO:teuthology.orchestra.run.smithi096.stdout: "val": 1844113099 2021-11-11T21:41:27.761 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.761 INFO:teuthology.orchestra.run.smithi096.stdout: ] 2021-11-11T21:41:27.761 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.761 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zonegroup": "test-zonegroup", 2021-11-11T21:41:27.762 INFO:teuthology.orchestra.run.smithi096.stdout: "master_zone": "badb2b8d-7821-4be0-b9ef-60c04d880ece", 2021-11-11T21:41:27.762 INFO:teuthology.orchestra.run.smithi096.stdout: "period_config": { 2021-11-11T21:41:27.762 INFO:teuthology.orchestra.run.smithi096.stdout: "bucket_quota": { 2021-11-11T21:41:27.762 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T21:41:27.763 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T21:41:27.763 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T21:41:27.763 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T21:41:27.763 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T21:41:27.763 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.764 INFO:teuthology.orchestra.run.smithi096.stdout: "user_quota": { 2021-11-11T21:41:27.764 INFO:teuthology.orchestra.run.smithi096.stdout: "enabled": false, 2021-11-11T21:41:27.764 INFO:teuthology.orchestra.run.smithi096.stdout: "check_on_raw": false, 2021-11-11T21:41:27.764 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size": -1, 2021-11-11T21:41:27.765 INFO:teuthology.orchestra.run.smithi096.stdout: "max_size_kb": 0, 2021-11-11T21:41:27.765 INFO:teuthology.orchestra.run.smithi096.stdout: "max_objects": -1 2021-11-11T21:41:27.765 INFO:teuthology.orchestra.run.smithi096.stdout: } 2021-11-11T21:41:27.765 INFO:teuthology.orchestra.run.smithi096.stdout: }, 2021-11-11T21:41:27.765 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_id": "31efe32c-6651-482e-86af-2519092e8c22", 2021-11-11T21:41:27.766 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_name": "test-realm", 2021-11-11T21:41:27.766 INFO:teuthology.orchestra.run.smithi096.stdout: "realm_epoch": 2 2021-11-11T21:41:27.766 INFO:teuthology.orchestra.run.smithi096.stdout:} 2021-11-11T21:41:27.767 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_zg_master_zone_delete ... ok 2021-11-11T21:41:27.767 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-46 2021-11-11T21:41:29.051 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-47 2021-11-11T21:41:29.127 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-48 2021-11-11T21:41:29.191 INFO:tasks.rgw.c1.client.1.smithi096.stdout:*** Caught signal (Segmentation fault) ** 2021-11-11T21:41:29.191 INFO:tasks.rgw.c1.client.1.smithi096.stdout: in thread 7fb5d8433700 thread_name:radosgw 2021-11-11T21:41:29.193 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-49 2021-11-11T21:41:29.203 INFO:tasks.rgw.c1.client.1.smithi096.stdout: ceph version 17.0.0-8977-ga3a8018b (a3a8018ba3cfe536252b397411bf18b7828c6098) quincy (dev) 2021-11-11T21:41:29.203 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 1: /lib64/libpthread.so.0(+0x12b20) [0x7fb6e5cbdb20] 2021-11-11T21:41:29.203 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 2: (OpsLogRados::log(req_state*, rgw_log_entry&)+0x123) [0x7fb6e88de7e3] 2021-11-11T21:41:29.204 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 3: (OpsLogManifold::log(req_state*, rgw_log_entry&)+0x3e) [0x7fb6e88dc77e] 2021-11-11T21:41:29.204 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 4: (rgw_log_op(RGWREST*, req_state*, std::__cxx11::basic_string, std::allocator > const&, OpsLogSink*)+0xadd) [0x7fb6e88df4fd] 2021-11-11T21:41:29.204 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 5: (process_request(rgw::sal::Store*, RGWREST*, RGWRequest*, std::__cxx11::basic_string, std::allocator > const&, rgw::auth::StrategyRegistry const&, RGWRestfulIO*, OpsLogSink*, optional_yield, rgw::dmclock::Scheduler*, std::__cxx11::basic_string, std::allocator >*, std::chrono::duration >*, int*)+0x1859) [0x7fb6e88fd9f9] 2021-11-11T21:41:29.204 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 6: /lib64/libradosgw.so.2(+0x54fcdd) [0x7fb6e882fcdd] 2021-11-11T21:41:29.205 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 7: /lib64/libradosgw.so.2(+0x5512c0) [0x7fb6e88312c0] 2021-11-11T21:41:29.205 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 8: /lib64/libradosgw.so.2(+0x55141c) [0x7fb6e883141c] 2021-11-11T21:41:29.205 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 9: make_fcontext() 2021-11-11T21:41:29.205 INFO:tasks.rgw.c1.client.1.smithi096.stdout:2021-11-11T21:41:29.191+0000 7fb5d8433700 -1 *** Caught signal (Segmentation fault) ** 2021-11-11T21:41:29.206 INFO:tasks.rgw.c1.client.1.smithi096.stdout: in thread 7fb5d8433700 thread_name:radosgw 2021-11-11T21:41:29.206 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 2021-11-11T21:41:29.207 INFO:tasks.rgw.c1.client.1.smithi096.stdout: ceph version 17.0.0-8977-ga3a8018b (a3a8018ba3cfe536252b397411bf18b7828c6098) quincy (dev) 2021-11-11T21:41:29.207 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 1: /lib64/libpthread.so.0(+0x12b20) [0x7fb6e5cbdb20] 2021-11-11T21:41:29.207 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 2: (OpsLogRados::log(req_state*, rgw_log_entry&)+0x123) [0x7fb6e88de7e3] 2021-11-11T21:41:29.207 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 3: (OpsLogManifold::log(req_state*, rgw_log_entry&)+0x3e) [0x7fb6e88dc77e] 2021-11-11T21:41:29.208 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 4: (rgw_log_op(RGWREST*, req_state*, std::__cxx11::basic_string, std::allocator > const&, OpsLogSink*)+0xadd) [0x7fb6e88df4fd] 2021-11-11T21:41:29.208 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 5: (process_request(rgw::sal::Store*, RGWREST*, RGWRequest*, std::__cxx11::basic_string, std::allocator > const&, rgw::auth::StrategyRegistry const&, RGWRestfulIO*, OpsLogSink*, optional_yield, rgw::dmclock::Scheduler*, std::__cxx11::basic_string, std::allocator >*, std::chrono::duration >*, int*)+0x1859) [0x7fb6e88fd9f9] 2021-11-11T21:41:29.208 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 6: /lib64/libradosgw.so.2(+0x54fcdd) [0x7fb6e882fcdd] 2021-11-11T21:41:29.209 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 7: /lib64/libradosgw.so.2(+0x5512c0) [0x7fb6e88312c0] 2021-11-11T21:41:29.209 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 8: /lib64/libradosgw.so.2(+0x55141c) [0x7fb6e883141c] 2021-11-11T21:41:29.209 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 9: make_fcontext() 2021-11-11T21:41:29.209 INFO:tasks.rgw.c1.client.1.smithi096.stdout: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2021-11-11T21:41:29.210 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 2021-11-11T21:41:29.212 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 0> 2021-11-11T21:41:29.191+0000 7fb5d8433700 -1 *** Caught signal (Segmentation fault) ** 2021-11-11T21:41:29.212 INFO:tasks.rgw.c1.client.1.smithi096.stdout: in thread 7fb5d8433700 thread_name:radosgw 2021-11-11T21:41:29.212 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 2021-11-11T21:41:29.213 INFO:tasks.rgw.c1.client.1.smithi096.stdout: ceph version 17.0.0-8977-ga3a8018b (a3a8018ba3cfe536252b397411bf18b7828c6098) quincy (dev) 2021-11-11T21:41:29.213 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 1: /lib64/libpthread.so.0(+0x12b20) [0x7fb6e5cbdb20] 2021-11-11T21:41:29.213 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 2: (OpsLogRados::log(req_state*, rgw_log_entry&)+0x123) [0x7fb6e88de7e3] 2021-11-11T21:41:29.214 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 3: (OpsLogManifold::log(req_state*, rgw_log_entry&)+0x3e) [0x7fb6e88dc77e] 2021-11-11T21:41:29.214 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 4: (rgw_log_op(RGWREST*, req_state*, std::__cxx11::basic_string, std::allocator > const&, OpsLogSink*)+0xadd) [0x7fb6e88df4fd] 2021-11-11T21:41:29.214 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 5: (process_request(rgw::sal::Store*, RGWREST*, RGWRequest*, std::__cxx11::basic_string, std::allocator > const&, rgw::auth::StrategyRegistry const&, RGWRestfulIO*, OpsLogSink*, optional_yield, rgw::dmclock::Scheduler*, std::__cxx11::basic_string, std::allocator >*, std::chrono::duration >*, int*)+0x1859) [0x7fb6e88fd9f9] 2021-11-11T21:41:29.214 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 6: /lib64/libradosgw.so.2(+0x54fcdd) [0x7fb6e882fcdd] 2021-11-11T21:41:29.215 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 7: /lib64/libradosgw.so.2(+0x5512c0) [0x7fb6e88312c0] 2021-11-11T21:41:29.215 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 8: /lib64/libradosgw.so.2(+0x55141c) [0x7fb6e883141c] 2021-11-11T21:41:29.215 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 9: make_fcontext() 2021-11-11T21:41:29.215 INFO:tasks.rgw.c1.client.1.smithi096.stdout: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2021-11-11T21:41:29.215 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 2021-11-11T21:41:29.226 INFO:tasks.rgw.c1.client.1.smithi096.stdout: -119> 2021-11-11T21:41:29.191+0000 7fb5d8433700 -1 *** Caught signal (Segmentation fault) ** 2021-11-11T21:41:29.227 INFO:tasks.rgw.c1.client.1.smithi096.stdout: in thread 7fb5d8433700 thread_name:radosgw 2021-11-11T21:41:29.227 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 2021-11-11T21:41:29.227 INFO:tasks.rgw.c1.client.1.smithi096.stdout: ceph version 17.0.0-8977-ga3a8018b (a3a8018ba3cfe536252b397411bf18b7828c6098) quincy (dev) 2021-11-11T21:41:29.228 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 1: /lib64/libpthread.so.0(+0x12b20) [0x7fb6e5cbdb20] 2021-11-11T21:41:29.228 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 2: (OpsLogRados::log(req_state*, rgw_log_entry&)+0x123) [0x7fb6e88de7e3] 2021-11-11T21:41:29.229 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 3: (OpsLogManifold::log(req_state*, rgw_log_entry&)+0x3e) [0x7fb6e88dc77e] 2021-11-11T21:41:29.229 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 4: (rgw_log_op(RGWREST*, req_state*, std::__cxx11::basic_string, std::allocator > const&, OpsLogSink*)+0xadd) [0x7fb6e88df4fd] 2021-11-11T21:41:29.229 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 5: (process_request(rgw::sal::Store*, RGWREST*, RGWRequest*, std::__cxx11::basic_string, std::allocator > const&, rgw::auth::StrategyRegistry const&, RGWRestfulIO*, OpsLogSink*, optional_yield, rgw::dmclock::Scheduler*, std::__cxx11::basic_string, std::allocator >*, std::chrono::duration >*, int*)+0x1859) [0x7fb6e88fd9f9] 2021-11-11T21:41:29.229 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 6: /lib64/libradosgw.so.2(+0x54fcdd) [0x7fb6e882fcdd] 2021-11-11T21:41:29.230 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 7: /lib64/libradosgw.so.2(+0x5512c0) [0x7fb6e88312c0] 2021-11-11T21:41:29.230 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 8: /lib64/libradosgw.so.2(+0x55141c) [0x7fb6e883141c] 2021-11-11T21:41:29.230 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 9: make_fcontext() 2021-11-11T21:41:29.230 INFO:tasks.rgw.c1.client.1.smithi096.stdout: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2021-11-11T21:41:29.231 INFO:tasks.rgw.c1.client.1.smithi096.stdout: 2021-11-11T21:41:29.245 INFO:rgw_multi.tests:create bucket zone=test-zone2 name=xdilif-50 2021-11-11T21:41:29.263 INFO:tasks.rgw.c1.client.1.smithi096.stderr:daemon-helper: command crashed with signal 11 2021-11-11T21:41:29.297 INFO:rgw_multi.tests:create bucket zone=test-zone3 name=xdilif-51 2021-11-11T21:41:31.647 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~0s 2021-11-11T21:41:37.252 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~6s 2021-11-11T21:41:42.858 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~11s 2021-11-11T21:41:48.464 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~17s 2021-11-11T21:41:54.071 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~22s 2021-11-11T21:41:59.676 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~28s 2021-11-11T21:42:05.283 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~34s 2021-11-11T21:42:10.888 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~39s 2021-11-11T21:42:16.493 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~45s 2021-11-11T21:42:22.100 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~50s 2021-11-11T21:42:27.706 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~56s 2021-11-11T21:42:33.311 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~62s 2021-11-11T21:42:38.918 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~67s 2021-11-11T21:42:44.525 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~73s 2021-11-11T21:42:50.131 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~78s 2021-11-11T21:42:55.737 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~84s 2021-11-11T21:43:01.344 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~90s 2021-11-11T21:43:06.950 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~95s 2021-11-11T21:43:12.557 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~101s 2021-11-11T21:43:18.163 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~107s 2021-11-11T21:43:23.769 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~112s 2021-11-11T21:43:29.375 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~118s 2021-11-11T21:43:34.981 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~123s 2021-11-11T21:43:40.587 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~129s 2021-11-11T21:43:46.193 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~135s 2021-11-11T21:43:51.799 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~140s 2021-11-11T21:43:57.405 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~146s 2021-11-11T21:44:03.010 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~151s 2021-11-11T21:44:08.616 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~157s 2021-11-11T21:44:14.222 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~163s 2021-11-11T21:44:19.828 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~168s 2021-11-11T21:44:25.432 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~174s 2021-11-11T21:44:31.038 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~179s 2021-11-11T21:44:36.644 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~185s 2021-11-11T21:44:42.249 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~191s 2021-11-11T21:44:47.855 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~196s 2021-11-11T21:44:53.460 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~202s 2021-11-11T21:44:59.066 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~207s 2021-11-11T21:45:04.672 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~213s 2021-11-11T21:45:10.278 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~219s 2021-11-11T21:45:15.883 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~224s 2021-11-11T21:45:21.489 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~230s 2021-11-11T21:45:27.095 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~235s 2021-11-11T21:45:32.699 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~241s 2021-11-11T21:45:38.305 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~247s 2021-11-11T21:45:43.909 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~252s 2021-11-11T21:45:49.514 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~258s 2021-11-11T21:45:55.120 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~263s 2021-11-11T21:46:00.725 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~269s 2021-11-11T21:46:06.330 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~275s 2021-11-11T21:46:11.935 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~280s 2021-11-11T21:46:17.540 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~286s 2021-11-11T21:46:23.146 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~291s 2021-11-11T21:46:28.751 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~297s 2021-11-11T21:46:34.356 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon c1.rgw.client.1 is failed for ~303s 2021-11-11T21:46:34.357 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons 2021-11-11T21:46:34.961 INFO:tasks.ceph.c1.osd.0:Sent signal 15 2021-11-11T21:46:34.964 INFO:tasks.ceph.c1.osd.1:Sent signal 15 2021-11-11T21:46:34.964 INFO:tasks.ceph.c1.osd.2:Sent signal 15 2021-11-11T21:46:34.965 INFO:tasks.ceph.c1.mon.a:Sent signal 15 2021-11-11T21:46:34.965 INFO:tasks.rgw.c1.client.0:Sent signal 15 2021-11-11T21:46:34.966 INFO:tasks.ceph.c1.mgr.x:Sent signal 15 2021-11-11T21:46:34.967 INFO:tasks.ceph.c1.osd.0.smithi096.stderr:2021-11-11T21:46:34.962+0000 7fd595f54700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster c1 -i 0 (PID: 32059) UID: 0 2021-11-11T21:46:34.967 INFO:tasks.ceph.c1.osd.0.smithi096.stderr:2021-11-11T21:46:34.962+0000 7fd595f54700 -1 osd.0 66 *** Got signal Terminated *** 2021-11-11T21:46:34.967 INFO:tasks.ceph.c1.osd.0.smithi096.stderr:2021-11-11T21:46:34.962+0000 7fd595f54700 -1 osd.0 66 *** Immediate shutdown (osd_fast_shutdown=true) *** 2021-11-11T21:46:34.968 INFO:tasks.ceph.c1.osd.1.smithi096.stderr:2021-11-11T21:46:34.964+0000 7fed7005d700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster c1 -i 1 (PID: 32062) UID: 0 2021-11-11T21:46:34.969 INFO:tasks.ceph.c1.osd.1.smithi096.stderr:2021-11-11T21:46:34.964+0000 7fed7005d700 -1 osd.1 66 *** Got signal Terminated *** 2021-11-11T21:46:34.969 INFO:tasks.ceph.c1.osd.1.smithi096.stderr:2021-11-11T21:46:34.964+0000 7fed7005d700 -1 osd.1 66 *** Immediate shutdown (osd_fast_shutdown=true) *** 2021-11-11T21:46:34.969 INFO:tasks.ceph.c1.osd.2.smithi096.stderr:2021-11-11T21:46:34.965+0000 7f7985884700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster c1 -i 2 (PID: 32064) UID: 0 2021-11-11T21:46:34.970 INFO:tasks.ceph.c1.osd.2.smithi096.stderr:2021-11-11T21:46:34.965+0000 7f7985884700 -1 osd.2 66 *** Got signal Terminated *** 2021-11-11T21:46:34.970 INFO:tasks.ceph.c1.osd.2.smithi096.stderr:2021-11-11T21:46:34.965+0000 7f7985884700 -1 osd.2 66 *** Immediate shutdown (osd_fast_shutdown=true) *** 2021-11-11T21:46:34.971 INFO:tasks.ceph.c1.mon.a.smithi096.stderr:2021-11-11T21:46:34.965+0000 7fdd11931700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-mon -f --cluster c1 -i a (PID: 31710) UID: 0 2021-11-11T21:46:34.971 INFO:tasks.ceph.c1.mon.a.smithi096.stderr:2021-11-11T21:46:34.965+0000 7fdd11931700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2021-11-11T21:46:34.972 INFO:tasks.rgw.c1.client.0.smithi096.stdout:2021-11-11T21:46:34.966+0000 7fb3784c7700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper term radosgw --rgw-frontends beast port=8000 -n client.0 --cluster c1 -k /etc/ceph/c1.client.0.keyring --log-file /var/log/ceph/rgw.c1.client.0.log --rgw_ops_log_socket_path /home/ubuntu/cephtest/rgw.opslog.c1.client.0.sock --foreground --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm (PID: 39120) UID: 0 2021-11-11T21:46:34.972 INFO:tasks.rgw.c1.client.0.smithi096.stdout:2021-11-11T21:46:34.966+0000 7fb389413440 -1 rgw dbstore: shutting down 2021-11-11T21:46:34.972 INFO:tasks.ceph.c1.mgr.x.smithi096.stderr:2021-11-11T21:46:34.966+0000 7f5d4281a700 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-mgr -f --cluster c1 -i x (PID: 31713) UID: 0 2021-11-11T21:46:34.973 INFO:tasks.ceph.c1.mgr.x.smithi096.stderr:2021-11-11T21:46:34.966+0000 7f5d4281a700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2021-11-11T21:49:47.092 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_set_bucket_website ... rgw_multi.tests.test_set_bucket_policy ... ERROR 2021-11-11T21:49:47.094 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-52 2021-11-11T21:59:13.323 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_sync_disable ... ERROR 2021-11-11T21:59:13.325 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-53 2021-11-11T22:08:15.912 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_sync_enable_right_after_disable ... ERROR 2021-11-11T22:08:15.915 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-54 2021-11-11T22:17:39.219 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_sync_disable_enable ... ERROR 2021-11-11T22:17:39.222 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-55 2021-11-11T22:26:43.497 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_multipart_object_sync ... ERROR 2021-11-11T22:26:43.527 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-56 2021-11-11T22:35:38.719 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_encrypted_object_sync ... ERROR 2021-11-11T22:35:38.722 INFO:rgw_multi.tests:create bucket zone=test-zone1 name=xdilif-57 2021-11-11T22:44:25.332 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_index_log_trim ... ERROR 2021-11-11T22:44:25.341 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T22:44:25.341 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T22:44:25.342 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T22:44:25.394 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T22:44:25.395 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T22:49:25.397 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T22:49:25.394+0000 7fc31e42f380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T22:54:25.397 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T22:54:25.395+0000 7fc31e42f380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T22:59:25.398 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T22:59:25.396+0000 7fc31e42f380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:04:25.400 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:04:25.397+0000 7fc31e42f380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:09:25.400 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:09:25.398+0000 7fc31e42f380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:14:25.401 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:14:25.399+0000 7fc31e42f380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:19:25.401 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:19:25.400+0000 7fc31e42f380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:24:25.402 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:24:25.400+0000 7fc31e42f380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:29:25.402 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:29:25.400+0000 7fc31e42f380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:34:25.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-11T23:34:25.406 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:34:25.400+0000 7fc31e42f380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:34:25.407 INFO:teuthology.orchestra.run.smithi096.stderr:failed to fetch mon config (--no-mon-config to skip) 2021-11-11T23:34:25.408 INFO:tasks.rgw_multisite_tests:rgw_multi.tests.test_bucket_creation_time ... FAIL 2021-11-11T23:34:25.409 INFO:tasks.rgw_multisite_tests:log information for manual testing ... ok 2021-11-11T23:34:25.410 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T23:34:25.414 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-11T23:34:25.415 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-11T23:34:25.465 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-11T23:34:25.466 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-11T23:39:25.469 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:39:25.466+0000 7f5ba3e58380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:44:25.469 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:44:25.467+0000 7f5ba3e58380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:49:25.469 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:49:25.467+0000 7f5ba3e58380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:54:25.470 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:54:25.469+0000 7f5ba3e58380 0 monclient(hunting): authenticate timed out after 300 2021-11-11T23:59:25.471 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-11T23:59:25.469+0000 7f5ba3e58380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:04:25.472 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:04:25.469+0000 7f5ba3e58380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:09:25.472 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:09:25.470+0000 7f5ba3e58380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:14:25.472 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:14:25.471+0000 7f5ba3e58380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:19:25.472 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:19:25.471+0000 7f5ba3e58380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:24:25.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-12T00:24:25.479 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:24:25.471+0000 7f5ba3e58380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:24:25.480 INFO:teuthology.orchestra.run.smithi096.stderr:failed to fetch mon config (--no-mon-config to skip) 2021-11-12T00:24:25.481 INFO:tasks.rgw_multisite_tests:test low level implementation of s3 notifications ... FAIL 2021-11-12T00:24:25.482 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T00:24:25.483 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T00:24:25.484 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-12T00:24:25.536 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-12T00:24:25.536 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-12T00:29:25.538 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:29:25.537+0000 7f49c6d31380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:34:25.537 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:34:25.537+0000 7f49c6d31380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:39:25.539 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:39:25.538+0000 7f49c6d31380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:44:25.538 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:44:25.538+0000 7f49c6d31380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:49:25.540 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:49:25.539+0000 7f49c6d31380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:54:25.540 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:54:25.539+0000 7f49c6d31380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T00:59:25.541 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T00:59:25.540+0000 7f49c6d31380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:04:25.541 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:04:25.541+0000 7f49c6d31380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:09:25.543 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:09:25.542+0000 7f49c6d31380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:14:25.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-12T01:14:25.553 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:14:25.542+0000 7f49c6d31380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:14:25.554 INFO:teuthology.orchestra.run.smithi096.stderr:failed to fetch mon config (--no-mon-config to skip) 2021-11-12T01:14:25.555 INFO:tasks.rgw_multisite_tests:test s3 records fetching ... FAIL 2021-11-12T01:14:25.556 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T01:14:25.557 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T01:14:25.558 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-12T01:14:25.608 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-12T01:14:25.609 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-12T01:19:25.611 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:19:25.610+0000 7f4ea3618380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:24:25.612 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:24:25.611+0000 7f4ea3618380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:29:25.612 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:29:25.611+0000 7f4ea3618380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:34:25.613 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:34:25.612+0000 7f4ea3618380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:39:25.613 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:39:25.613+0000 7f4ea3618380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:44:25.614 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:44:25.612+0000 7f4ea3618380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:49:25.615 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:49:25.613+0000 7f4ea3618380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:54:25.615 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:54:25.614+0000 7f4ea3618380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T01:59:25.616 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T01:59:25.615+0000 7f4ea3618380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:04:25.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-12T02:04:25.622 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:04:25.615+0000 7f4ea3618380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:04:25.622 INFO:teuthology.orchestra.run.smithi096.stderr:failed to fetch mon config (--no-mon-config to skip) 2021-11-12T02:04:25.624 INFO:tasks.rgw_multisite_tests:test s3 notification set/get/delete ... FAIL 2021-11-12T02:04:25.625 INFO:tasks.rgw_multisite_tests:test s3 notification filter on master ... ok 2021-11-12T02:04:25.625 INFO:tasks.rgw_multisite_tests:rgw_multi.tests_ps.test_object_timing ... ok 2021-11-12T02:04:25.626 INFO:tasks.rgw_multisite_tests:test that opaque id set in topic, is sent in notification ... ok 2021-11-12T02:04:25.627 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T02:04:25.628 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T02:04:25.629 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-12T02:04:25.677 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-12T02:04:25.678 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-12T02:09:25.680 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:09:25.679+0000 7fefe189c380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:14:25.682 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:14:25.680+0000 7fefe189c380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:19:25.683 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:19:25.681+0000 7fefe189c380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:24:25.683 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:24:25.682+0000 7fefe189c380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:29:25.684 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:29:25.682+0000 7fefe189c380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:34:25.685 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:34:25.682+0000 7fefe189c380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:39:25.687 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:39:25.683+0000 7fefe189c380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:44:25.685 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:44:25.684+0000 7fefe189c380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:49:25.685 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:49:25.684+0000 7fefe189c380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:54:25.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-12T02:54:25.722 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:54:25.684+0000 7fefe189c380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T02:54:25.722 INFO:teuthology.orchestra.run.smithi096.stderr:failed to fetch mon config (--no-mon-config to skip) 2021-11-12T02:54:25.724 INFO:tasks.rgw_multisite_tests:test set/get/delete of topic ... FAIL 2021-11-12T02:54:25.725 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T02:54:25.725 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T02:54:25.726 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-12T02:54:25.778 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-12T02:54:25.778 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-12T02:59:25.780 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T02:59:25.778+0000 7f799b1c5380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:04:25.781 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:04:25.779+0000 7f799b1c5380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:09:25.782 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:09:25.780+0000 7f799b1c5380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:14:25.782 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:14:25.780+0000 7f799b1c5380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:19:25.783 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:19:25.781+0000 7f799b1c5380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:24:25.783 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:24:25.782+0000 7f799b1c5380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:29:25.784 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:29:25.782+0000 7f799b1c5380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:34:25.785 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:34:25.782+0000 7f799b1c5380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:39:25.785 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:39:25.783+0000 7f799b1c5380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:44:25.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-12T03:44:25.796 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:44:25.783+0000 7f799b1c5380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:44:25.796 INFO:teuthology.orchestra.run.smithi096.stderr:failed to fetch mon config (--no-mon-config to skip) 2021-11-12T03:44:25.798 INFO:tasks.rgw_multisite_tests:test set topic with endpoint ... FAIL 2021-11-12T03:44:25.799 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T03:44:25.800 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T03:44:25.801 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-12T03:44:25.851 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-12T03:44:25.852 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-12T03:49:25.854 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:49:25.851+0000 7f1492c7d380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:54:25.854 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:54:25.852+0000 7f1492c7d380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T03:59:25.854 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T03:59:25.852+0000 7f1492c7d380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:04:25.855 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:04:25.853+0000 7f1492c7d380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:09:25.856 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:09:25.854+0000 7f1492c7d380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:14:25.856 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:14:25.855+0000 7f1492c7d380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:19:25.857 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:19:25.855+0000 7f1492c7d380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:24:25.857 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:24:25.855+0000 7f1492c7d380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:29:25.858 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:29:25.856+0000 7f1492c7d380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:34:25.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-12T04:34:25.895 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:34:25.856+0000 7f1492c7d380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:34:25.896 INFO:teuthology.orchestra.run.smithi096.stderr:failed to fetch mon config (--no-mon-config to skip) 2021-11-12T04:34:25.897 INFO:tasks.rgw_multisite_tests:test set/get/delete of notification ... FAIL 2021-11-12T04:34:25.898 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T04:34:25.899 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T04:34:25.900 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-12T04:34:25.949 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-12T04:34:25.950 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-12T04:39:25.952 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:39:25.950+0000 7efe3c587380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:44:25.953 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:44:25.950+0000 7efe3c587380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:49:25.953 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:49:25.951+0000 7efe3c587380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:54:25.954 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:54:25.950+0000 7efe3c587380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T04:59:25.954 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T04:59:25.952+0000 7efe3c587380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:04:25.954 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:04:25.952+0000 7efe3c587380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:09:25.955 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:09:25.953+0000 7efe3c587380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:14:25.955 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:14:25.953+0000 7efe3c587380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:19:25.956 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:19:25.953+0000 7efe3c587380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:24:25.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-12T05:24:25.969 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:24:25.954+0000 7efe3c587380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:24:25.969 INFO:teuthology.orchestra.run.smithi096.stderr:failed to fetch mon config (--no-mon-config to skip) 2021-11-12T05:24:25.971 INFO:tasks.rgw_multisite_tests:test set/get/delete of notification on specific events ... FAIL 2021-11-12T05:24:25.972 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T05:24:25.972 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T05:24:25.973 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-12T05:24:26.023 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-12T05:24:26.024 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-12T05:29:26.026 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:29:26.025+0000 7f6c32963380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:34:26.026 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:34:26.025+0000 7f6c32963380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:39:26.027 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:39:26.026+0000 7f6c32963380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:44:26.028 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:44:26.026+0000 7f6c32963380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:49:26.028 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:49:26.027+0000 7f6c32963380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:54:26.029 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:54:26.027+0000 7f6c32963380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T05:59:26.029 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T05:59:26.027+0000 7f6c32963380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:04:26.029 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:04:26.027+0000 7f6c32963380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:09:26.030 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:09:26.028+0000 7f6c32963380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:14:26.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-12T06:14:26.034 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:14:26.029+0000 7f6c32963380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:14:26.034 INFO:teuthology.orchestra.run.smithi096.stderr:failed to fetch mon config (--no-mon-config to skip) 2021-11-12T06:14:26.036 INFO:tasks.rgw_multisite_tests:test set/get/delete of subscription ... FAIL 2021-11-12T06:14:26.037 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T06:14:26.038 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T06:14:26.038 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-12T06:14:26.089 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-12T06:14:26.089 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-12T06:19:26.092 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:19:26.090+0000 7f9baf8f2380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:24:26.093 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:24:26.091+0000 7f9baf8f2380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:29:26.093 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:29:26.092+0000 7f9baf8f2380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:34:26.094 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:34:26.092+0000 7f9baf8f2380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:39:26.094 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:39:26.093+0000 7f9baf8f2380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:44:26.094 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:44:26.093+0000 7f9baf8f2380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:49:26.095 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:49:26.093+0000 7f9baf8f2380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:54:26.096 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:54:26.094+0000 7f9baf8f2380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T06:59:26.096 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T06:59:26.094+0000 7f9baf8f2380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:04:26.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-12T07:04:26.111 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:04:26.095+0000 7f9baf8f2380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:04:26.111 INFO:teuthology.orchestra.run.smithi096.stderr:failed to fetch mon config (--no-mon-config to skip) 2021-11-12T07:04:26.113 INFO:tasks.rgw_multisite_tests:test radosgw-admin commands ... FAIL 2021-11-12T07:04:26.115 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T07:04:26.115 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T07:04:26.116 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-12T07:04:26.167 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-12T07:04:26.167 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-12T07:09:26.170 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:09:26.169+0000 7fa745f54380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:14:26.171 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:14:26.169+0000 7fa745f54380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:19:26.171 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:19:26.170+0000 7fa745f54380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:24:26.171 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:24:26.170+0000 7fa745f54380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:29:26.172 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:29:26.171+0000 7fa745f54380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:34:26.173 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:34:26.171+0000 7fa745f54380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:39:26.173 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:39:26.172+0000 7fa745f54380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:44:26.173 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:44:26.172+0000 7fa745f54380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:49:26.174 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:49:26.173+0000 7fa745f54380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:54:26.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2021-11-12T07:54:26.178 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:54:26.173+0000 7fa745f54380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T07:54:26.178 INFO:teuthology.orchestra.run.smithi096.stderr:failed to fetch mon config (--no-mon-config to skip) 2021-11-12T07:54:26.179 INFO:tasks.rgw_multisite_tests:test that events are only sent on incremental sync ... FAIL 2021-11-12T07:54:26.181 INFO:tasks.util.rgw:rgwadmin: c1.client.0 : ['mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T07:54:26.181 DEBUG:tasks.util.rgw:rgwadmin: cmd=['adjust-ulimits', 'ceph-coverage', '/home/ubuntu/cephtest/archive/coverage', 'radosgw-admin', '--log-to-stderr', '--format', 'json', '-n', 'client.0', '--cluster', 'c1', 'mdlog', 'status', '--rgw-zone', 'test-zone1', '--rgw-zonegroup', 'test-zonegroup', '--rgw-realm', 'test-realm', '--cluster', 'c1', '--debug-rgw', '0', '--debug-ms', '0', '--rgw-cache-enabled', 'false'] 2021-11-12T07:54:26.182 DEBUG:teuthology.orchestra.run.smithi096:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage radosgw-admin --log-to-stderr --format json -n client.0 --cluster c1 mdlog status --rgw-zone test-zone1 --rgw-zonegroup test-zonegroup --rgw-realm test-realm --cluster c1 --debug-rgw 0 --debug-ms 0 --rgw-cache-enabled false 2021-11-12T07:54:26.232 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setuser ceph since I am not root 2021-11-12T07:54:26.232 INFO:teuthology.orchestra.run.smithi096.stderr:ignoring --setgroup ceph since I am not root 2021-11-12T07:59:26.236 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T07:59:26.234+0000 7fb411307380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T08:04:26.235 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T08:04:26.234+0000 7fb411307380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T08:09:26.237 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T08:09:26.235+0000 7fb411307380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T08:14:26.237 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T08:14:26.234+0000 7fb411307380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T08:19:26.241 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T08:19:26.235+0000 7fb411307380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T08:24:26.238 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T08:24:26.236+0000 7fb411307380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T08:29:26.238 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T08:29:26.237+0000 7fb411307380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T08:34:26.239 INFO:teuthology.orchestra.run.smithi096.stderr:2021-11-12T08:34:26.237+0000 7fb411307380 0 monclient(hunting): authenticate timed out after 300 2021-11-12T08:37:35.772 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2021-11-12T08:37:35.778 DEBUG:teuthology.task.console_log:Killing console logger for smithi096 2021-11-12T08:37:35.779 DEBUG:teuthology.task.console_log:Killing console logger for smithi198 2021-11-12T08:37:35.780 DEBUG:teuthology.exit:Finished running handlers